| Line Number |
../DebugInfoTest/example_mips_dbg.ll
BUT NOT
../DebugInfoTest/example_mips.ll
|
Line Number |
../DebugInfoTest/example_mips.ll
BUT NOT
../DebugInfoTest/example_mips_dbg.ll
|
| 1 |
//===-- PPCISelDAGToDAG.cpp - PPC --pattern matching inst selector --------===// |
1 |
//===-- PPCISelDAGToDAG.cpp - PPC --pattern matching inst selector --------===// |
| 2 |
// |
2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
7 |
//===----------------------------------------------------------------------===// |
| 8 |
// |
8 |
// |
| 9 |
// This file defines a pattern matching instruction selector for PowerPC, |
9 |
// This file defines a pattern matching instruction selector for PowerPC, |
| 10 |
// converting from a legalized dag to a PPC dag. |
10 |
// converting from a legalized dag to a PPC dag. |
| 11 |
// |
11 |
// |
| 12 |
//===----------------------------------------------------------------------===// |
12 |
//===----------------------------------------------------------------------===// |
| 13 |
|
13 |
|
| 14 |
#include "MCTargetDesc/PPCMCTargetDesc.h" |
14 |
#include "MCTargetDesc/PPCMCTargetDesc.h" |
| 15 |
#include "MCTargetDesc/PPCPredicates.h" |
15 |
#include "MCTargetDesc/PPCPredicates.h" |
| 16 |
#include "PPC.h" |
16 |
#include "PPC.h" |
| 17 |
#include "PPCISelLowering.h" |
17 |
#include "PPCISelLowering.h" |
| 18 |
#include "PPCMachineFunctionInfo.h" |
18 |
#include "PPCMachineFunctionInfo.h" |
| 19 |
#include "PPCSubtarget.h" |
19 |
#include "PPCSubtarget.h" |
| 20 |
#include "PPCTargetMachine.h" |
20 |
#include "PPCTargetMachine.h" |
| 21 |
#include "llvm/ADT/APInt.h" |
21 |
#include "llvm/ADT/APInt.h" |
| 22 |
#include "llvm/ADT/APSInt.h" |
22 |
#include "llvm/ADT/APSInt.h" |
| 23 |
#include "llvm/ADT/DenseMap.h" |
23 |
#include "llvm/ADT/DenseMap.h" |
| 24 |
#include "llvm/ADT/STLExtras.h" |
24 |
#include "llvm/ADT/STLExtras.h" |
| 25 |
#include "llvm/ADT/SmallPtrSet.h" |
25 |
#include "llvm/ADT/SmallPtrSet.h" |
| 26 |
#include "llvm/ADT/SmallVector.h" |
26 |
#include "llvm/ADT/SmallVector.h" |
| 27 |
#include "llvm/ADT/Statistic.h" |
27 |
#include "llvm/ADT/Statistic.h" |
| 28 |
#include "llvm/Analysis/BranchProbabilityInfo.h" |
28 |
#include "llvm/Analysis/BranchProbabilityInfo.h" |
| 29 |
#include "llvm/CodeGen/FunctionLoweringInfo.h" |
29 |
#include "llvm/CodeGen/FunctionLoweringInfo.h" |
| 30 |
#include "llvm/CodeGen/ISDOpcodes.h" |
30 |
#include "llvm/CodeGen/ISDOpcodes.h" |
| 31 |
#include "llvm/CodeGen/MachineBasicBlock.h" |
31 |
#include "llvm/CodeGen/MachineBasicBlock.h" |
| 32 |
#include "llvm/CodeGen/MachineFrameInfo.h" |
32 |
#include "llvm/CodeGen/MachineFrameInfo.h" |
| 33 |
#include "llvm/CodeGen/MachineFunction.h" |
33 |
#include "llvm/CodeGen/MachineFunction.h" |
| 34 |
#include "llvm/CodeGen/MachineInstrBuilder.h" |
34 |
#include "llvm/CodeGen/MachineInstrBuilder.h" |
| 35 |
#include "llvm/CodeGen/MachineRegisterInfo.h" |
35 |
#include "llvm/CodeGen/MachineRegisterInfo.h" |
| 36 |
#include "llvm/CodeGen/MachineValueType.h" |
36 |
#include "llvm/CodeGen/MachineValueType.h" |
| 37 |
#include "llvm/CodeGen/SelectionDAG.h" |
37 |
#include "llvm/CodeGen/SelectionDAG.h" |
| 38 |
#include "llvm/CodeGen/SelectionDAGISel.h" |
38 |
#include "llvm/CodeGen/SelectionDAGISel.h" |
| 39 |
#include "llvm/CodeGen/SelectionDAGNodes.h" |
39 |
#include "llvm/CodeGen/SelectionDAGNodes.h" |
| 40 |
#include "llvm/CodeGen/TargetInstrInfo.h" |
40 |
#include "llvm/CodeGen/TargetInstrInfo.h" |
| 41 |
#include "llvm/CodeGen/TargetRegisterInfo.h" |
41 |
#include "llvm/CodeGen/TargetRegisterInfo.h" |
| 42 |
#include "llvm/CodeGen/ValueTypes.h" |
42 |
#include "llvm/CodeGen/ValueTypes.h" |
| 43 |
#include "llvm/IR/BasicBlock.h" |
43 |
#include "llvm/IR/BasicBlock.h" |
| 44 |
#include "llvm/IR/DebugLoc.h" |
44 |
#include "llvm/IR/DebugLoc.h" |
| 45 |
#include "llvm/IR/Function.h" |
45 |
#include "llvm/IR/Function.h" |
| 46 |
#include "llvm/IR/GlobalValue.h" |
46 |
#include "llvm/IR/GlobalValue.h" |
| 47 |
#include "llvm/IR/InlineAsm.h" |
47 |
#include "llvm/IR/InlineAsm.h" |
| 48 |
#include "llvm/IR/InstrTypes.h" |
48 |
#include "llvm/IR/InstrTypes.h" |
| 49 |
#include "llvm/IR/IntrinsicsPowerPC.h" |
49 |
#include "llvm/IR/IntrinsicsPowerPC.h" |
| 50 |
#include "llvm/IR/Module.h" |
50 |
#include "llvm/IR/Module.h" |
| 51 |
#include "llvm/Support/Casting.h" |
51 |
#include "llvm/Support/Casting.h" |
| 52 |
#include "llvm/Support/CodeGen.h" |
52 |
#include "llvm/Support/CodeGen.h" |
| 53 |
#include "llvm/Support/CommandLine.h" |
53 |
#include "llvm/Support/CommandLine.h" |
| 54 |
#include "llvm/Support/Compiler.h" |
54 |
#include "llvm/Support/Compiler.h" |
| 55 |
#include "llvm/Support/Debug.h" |
55 |
#include "llvm/Support/Debug.h" |
| 56 |
#include "llvm/Support/ErrorHandling.h" |
56 |
#include "llvm/Support/ErrorHandling.h" |
| 57 |
#include "llvm/Support/KnownBits.h" |
57 |
#include "llvm/Support/KnownBits.h" |
| 58 |
#include "llvm/Support/MathExtras.h" |
58 |
#include "llvm/Support/MathExtras.h" |
| 59 |
#include "llvm/Support/raw_ostream.h" |
59 |
#include "llvm/Support/raw_ostream.h" |
| 60 |
#include |
60 |
#include |
| 61 |
#include |
61 |
#include |
| 62 |
#include |
62 |
#include |
| 63 |
#include |
63 |
#include |
| 64 |
#include |
64 |
#include |
| 65 |
#include |
65 |
#include |
| 66 |
#include |
66 |
#include |
| 67 |
#include |
67 |
#include |
| 68 |
#include |
68 |
#include |
| 69 |
|
69 |
|
| 70 |
using namespace llvm; |
70 |
using namespace llvm; |
| 71 |
|
71 |
|
| 72 |
#define DEBUG_TYPE "ppc-isel" |
72 |
#define DEBUG_TYPE "ppc-isel" |
| 73 |
#define PASS_NAME "PowerPC DAG->DAG Pattern Instruction Selection" |
73 |
#define PASS_NAME "PowerPC DAG->DAG Pattern Instruction Selection" |
| 74 |
|
74 |
|
| 75 |
STATISTIC(NumSextSetcc, |
75 |
STATISTIC(NumSextSetcc, |
| 76 |
"Number of (sext(setcc)) nodes expanded into GPR sequence."); |
76 |
"Number of (sext(setcc)) nodes expanded into GPR sequence."); |
| 77 |
STATISTIC(NumZextSetcc, |
77 |
STATISTIC(NumZextSetcc, |
| 78 |
"Number of (zext(setcc)) nodes expanded into GPR sequence."); |
78 |
"Number of (zext(setcc)) nodes expanded into GPR sequence."); |
| 79 |
STATISTIC(SignExtensionsAdded, |
79 |
STATISTIC(SignExtensionsAdded, |
| 80 |
"Number of sign extensions for compare inputs added."); |
80 |
"Number of sign extensions for compare inputs added."); |
| 81 |
STATISTIC(ZeroExtensionsAdded, |
81 |
STATISTIC(ZeroExtensionsAdded, |
| 82 |
"Number of zero extensions for compare inputs added."); |
82 |
"Number of zero extensions for compare inputs added."); |
| 83 |
STATISTIC(NumLogicOpsOnComparison, |
83 |
STATISTIC(NumLogicOpsOnComparison, |
| 84 |
"Number of logical ops on i1 values calculated in GPR."); |
84 |
"Number of logical ops on i1 values calculated in GPR."); |
| 85 |
STATISTIC(OmittedForNonExtendUses, |
85 |
STATISTIC(OmittedForNonExtendUses, |
| 86 |
"Number of compares not eliminated as they have non-extending uses."); |
86 |
"Number of compares not eliminated as they have non-extending uses."); |
| 87 |
STATISTIC(NumP9Setb, |
87 |
STATISTIC(NumP9Setb, |
| 88 |
"Number of compares lowered to setb."); |
88 |
"Number of compares lowered to setb."); |
| 89 |
|
89 |
|
| 90 |
// FIXME: Remove this once the bug has been fixed! |
90 |
// FIXME: Remove this once the bug has been fixed! |
| 91 |
cl::opt ANDIGlueBug("expose-ppc-andi-glue-bug", |
91 |
cl::opt ANDIGlueBug("expose-ppc-andi-glue-bug", |
| 92 |
cl::desc("expose the ANDI glue bug on PPC"), cl::Hidden); |
92 |
cl::desc("expose the ANDI glue bug on PPC"), cl::Hidden); |
| 93 |
|
93 |
|
| 94 |
static cl::opt |
94 |
static cl::opt |
| 95 |
UseBitPermRewriter("ppc-use-bit-perm-rewriter", cl::init(true), |
95 |
UseBitPermRewriter("ppc-use-bit-perm-rewriter", cl::init(true), |
| 96 |
cl::desc("use aggressive ppc isel for bit permutations"), |
96 |
cl::desc("use aggressive ppc isel for bit permutations"), |
| 97 |
cl::Hidden); |
97 |
cl::Hidden); |
| 98 |
static cl::opt BPermRewriterNoMasking( |
98 |
static cl::opt BPermRewriterNoMasking( |
| 99 |
"ppc-bit-perm-rewriter-stress-rotates", |
99 |
"ppc-bit-perm-rewriter-stress-rotates", |
| 100 |
cl::desc("stress rotate selection in aggressive ppc isel for " |
100 |
cl::desc("stress rotate selection in aggressive ppc isel for " |
| 101 |
"bit permutations"), |
101 |
"bit permutations"), |
| 102 |
cl::Hidden); |
102 |
cl::Hidden); |
| 103 |
|
103 |
|
| 104 |
static cl::opt EnableBranchHint( |
104 |
static cl::opt EnableBranchHint( |
| 105 |
"ppc-use-branch-hint", cl::init(true), |
105 |
"ppc-use-branch-hint", cl::init(true), |
| 106 |
cl::desc("Enable static hinting of branches on ppc"), |
106 |
cl::desc("Enable static hinting of branches on ppc"), |
| 107 |
cl::Hidden); |
107 |
cl::Hidden); |
| 108 |
|
108 |
|
| 109 |
static cl::opt EnableTLSOpt( |
109 |
static cl::opt EnableTLSOpt( |
| 110 |
"ppc-tls-opt", cl::init(true), |
110 |
"ppc-tls-opt", cl::init(true), |
| 111 |
cl::desc("Enable tls optimization peephole"), |
111 |
cl::desc("Enable tls optimization peephole"), |
| 112 |
cl::Hidden); |
112 |
cl::Hidden); |
| 113 |
|
113 |
|
| 114 |
enum ICmpInGPRType { ICGPR_All, ICGPR_None, ICGPR_I32, ICGPR_I64, |
114 |
enum ICmpInGPRType { ICGPR_All, ICGPR_None, ICGPR_I32, ICGPR_I64, |
| 115 |
ICGPR_NonExtIn, ICGPR_Zext, ICGPR_Sext, ICGPR_ZextI32, |
115 |
ICGPR_NonExtIn, ICGPR_Zext, ICGPR_Sext, ICGPR_ZextI32, |
| 116 |
ICGPR_SextI32, ICGPR_ZextI64, ICGPR_SextI64 }; |
116 |
ICGPR_SextI32, ICGPR_ZextI64, ICGPR_SextI64 }; |
| 117 |
|
117 |
|
| 118 |
static cl::opt CmpInGPR( |
118 |
static cl::opt CmpInGPR( |
| 119 |
"ppc-gpr-icmps", cl::Hidden, cl::init(ICGPR_All), |
119 |
"ppc-gpr-icmps", cl::Hidden, cl::init(ICGPR_All), |
| 120 |
cl::desc("Specify the types of comparisons to emit GPR-only code for."), |
120 |
cl::desc("Specify the types of comparisons to emit GPR-only code for."), |
| 121 |
cl::values(clEnumValN(ICGPR_None, "none", "Do not modify integer comparisons."), |
121 |
cl::values(clEnumValN(ICGPR_None, "none", "Do not modify integer comparisons."), |
| 122 |
clEnumValN(ICGPR_All, "all", "All possible int comparisons in GPRs."), |
122 |
clEnumValN(ICGPR_All, "all", "All possible int comparisons in GPRs."), |
| 123 |
clEnumValN(ICGPR_I32, "i32", "Only i32 comparisons in GPRs."), |
123 |
clEnumValN(ICGPR_I32, "i32", "Only i32 comparisons in GPRs."), |
| 124 |
clEnumValN(ICGPR_I64, "i64", "Only i64 comparisons in GPRs."), |
124 |
clEnumValN(ICGPR_I64, "i64", "Only i64 comparisons in GPRs."), |
| 125 |
clEnumValN(ICGPR_NonExtIn, "nonextin", |
125 |
clEnumValN(ICGPR_NonExtIn, "nonextin", |
| 126 |
"Only comparisons where inputs don't need [sz]ext."), |
126 |
"Only comparisons where inputs don't need [sz]ext."), |
| 127 |
clEnumValN(ICGPR_Zext, "zext", "Only comparisons with zext result."), |
127 |
clEnumValN(ICGPR_Zext, "zext", "Only comparisons with zext result."), |
| 128 |
clEnumValN(ICGPR_ZextI32, "zexti32", |
128 |
clEnumValN(ICGPR_ZextI32, "zexti32", |
| 129 |
"Only i32 comparisons with zext result."), |
129 |
"Only i32 comparisons with zext result."), |
| 130 |
clEnumValN(ICGPR_ZextI64, "zexti64", |
130 |
clEnumValN(ICGPR_ZextI64, "zexti64", |
| 131 |
"Only i64 comparisons with zext result."), |
131 |
"Only i64 comparisons with zext result."), |
| 132 |
clEnumValN(ICGPR_Sext, "sext", "Only comparisons with sext result."), |
132 |
clEnumValN(ICGPR_Sext, "sext", "Only comparisons with sext result."), |
| 133 |
clEnumValN(ICGPR_SextI32, "sexti32", |
133 |
clEnumValN(ICGPR_SextI32, "sexti32", |
| 134 |
"Only i32 comparisons with sext result."), |
134 |
"Only i32 comparisons with sext result."), |
| 135 |
clEnumValN(ICGPR_SextI64, "sexti64", |
135 |
clEnumValN(ICGPR_SextI64, "sexti64", |
| 136 |
"Only i64 comparisons with sext result."))); |
136 |
"Only i64 comparisons with sext result."))); |
| 137 |
namespace { |
137 |
namespace { |
| 138 |
|
138 |
|
| 139 |
//===--------------------------------------------------------------------===// |
139 |
//===--------------------------------------------------------------------===// |
| 140 |
/// PPCDAGToDAGISel - PPC specific code to select PPC machine |
140 |
/// PPCDAGToDAGISel - PPC specific code to select PPC machine |
| 141 |
/// instructions for SelectionDAG operations. |
141 |
/// instructions for SelectionDAG operations. |
| 142 |
/// |
142 |
/// |
| 143 |
class PPCDAGToDAGISel : public SelectionDAGISel { |
143 |
class PPCDAGToDAGISel : public SelectionDAGISel { |
| 144 |
const PPCTargetMachine &TM; |
144 |
const PPCTargetMachine &TM; |
| 145 |
const PPCSubtarget *Subtarget = nullptr; |
145 |
const PPCSubtarget *Subtarget = nullptr; |
| 146 |
const PPCTargetLowering *PPCLowering = nullptr; |
146 |
const PPCTargetLowering *PPCLowering = nullptr; |
| 147 |
unsigned GlobalBaseReg = 0; |
147 |
unsigned GlobalBaseReg = 0; |
| 148 |
|
148 |
|
| 149 |
public: |
149 |
public: |
| 150 |
static char ID; |
150 |
static char ID; |
| 151 |
|
151 |
|
| 152 |
PPCDAGToDAGISel() = delete; |
152 |
PPCDAGToDAGISel() = delete; |
| 153 |
|
153 |
|
| 154 |
explicit PPCDAGToDAGISel(PPCTargetMachine &tm, CodeGenOpt::Level OptLevel) |
154 |
explicit PPCDAGToDAGISel(PPCTargetMachine &tm, CodeGenOpt::Level OptLevel) |
| 155 |
: SelectionDAGISel(ID, tm, OptLevel), TM(tm) {} |
155 |
: SelectionDAGISel(ID, tm, OptLevel), TM(tm) {} |
| 156 |
|
156 |
|
| 157 |
bool runOnMachineFunction(MachineFunction &MF) override { |
157 |
bool runOnMachineFunction(MachineFunction &MF) override { |
| 158 |
// Make sure we re-emit a set of the global base reg if necessary |
158 |
// Make sure we re-emit a set of the global base reg if necessary |
| 159 |
GlobalBaseReg = 0; |
159 |
GlobalBaseReg = 0; |
| 160 |
Subtarget = &MF.getSubtarget(); |
160 |
Subtarget = &MF.getSubtarget(); |
| 161 |
PPCLowering = Subtarget->getTargetLowering(); |
161 |
PPCLowering = Subtarget->getTargetLowering(); |
| 162 |
if (Subtarget->hasROPProtect()) { |
162 |
if (Subtarget->hasROPProtect()) { |
| 163 |
// Create a place on the stack for the ROP Protection Hash. |
163 |
// Create a place on the stack for the ROP Protection Hash. |
| 164 |
// The ROP Protection Hash will always be 8 bytes and aligned to 8 |
164 |
// The ROP Protection Hash will always be 8 bytes and aligned to 8 |
| 165 |
// bytes. |
165 |
// bytes. |
| 166 |
MachineFrameInfo &MFI = MF.getFrameInfo(); |
166 |
MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 167 |
PPCFunctionInfo *FI = MF.getInfo(); |
167 |
PPCFunctionInfo *FI = MF.getInfo(); |
| 168 |
const int Result = MFI.CreateStackObject(8, Align(8), false); |
168 |
const int Result = MFI.CreateStackObject(8, Align(8), false); |
| 169 |
FI->setROPProtectionHashSaveIndex(Result); |
169 |
FI->setROPProtectionHashSaveIndex(Result); |
| 170 |
} |
170 |
} |
| 171 |
SelectionDAGISel::runOnMachineFunction(MF); |
171 |
SelectionDAGISel::runOnMachineFunction(MF); |
| 172 |
|
172 |
|
| 173 |
return true; |
173 |
return true; |
| 174 |
} |
174 |
} |
| 175 |
|
175 |
|
| 176 |
void PreprocessISelDAG() override; |
176 |
void PreprocessISelDAG() override; |
| 177 |
void PostprocessISelDAG() override; |
177 |
void PostprocessISelDAG() override; |
| 178 |
|
178 |
|
| 179 |
/// getI16Imm - Return a target constant with the specified value, of type |
179 |
/// getI16Imm - Return a target constant with the specified value, of type |
| 180 |
/// i16. |
180 |
/// i16. |
| 181 |
inline SDValue getI16Imm(unsigned Imm, const SDLoc &dl) { |
181 |
inline SDValue getI16Imm(unsigned Imm, const SDLoc &dl) { |
| 182 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i16); |
182 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i16); |
| 183 |
} |
183 |
} |
| 184 |
|
184 |
|
| 185 |
/// getI32Imm - Return a target constant with the specified value, of type |
185 |
/// getI32Imm - Return a target constant with the specified value, of type |
| 186 |
/// i32. |
186 |
/// i32. |
| 187 |
inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) { |
187 |
inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) { |
| 188 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
188 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
| 189 |
} |
189 |
} |
| 190 |
|
190 |
|
| 191 |
/// getI64Imm - Return a target constant with the specified value, of type |
191 |
/// getI64Imm - Return a target constant with the specified value, of type |
| 192 |
/// i64. |
192 |
/// i64. |
| 193 |
inline SDValue getI64Imm(uint64_t Imm, const SDLoc &dl) { |
193 |
inline SDValue getI64Imm(uint64_t Imm, const SDLoc &dl) { |
| 194 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i64); |
194 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i64); |
| 195 |
} |
195 |
} |
| 196 |
|
196 |
|
| 197 |
/// getSmallIPtrImm - Return a target constant of pointer type. |
197 |
/// getSmallIPtrImm - Return a target constant of pointer type. |
| 198 |
inline SDValue getSmallIPtrImm(uint64_t Imm, const SDLoc &dl) { |
198 |
inline SDValue getSmallIPtrImm(uint64_t Imm, const SDLoc &dl) { |
| 199 |
return CurDAG->getTargetConstant( |
199 |
return CurDAG->getTargetConstant( |
| 200 |
Imm, dl, PPCLowering->getPointerTy(CurDAG->getDataLayout())); |
200 |
Imm, dl, PPCLowering->getPointerTy(CurDAG->getDataLayout())); |
| 201 |
} |
201 |
} |
| 202 |
|
202 |
|
| 203 |
/// isRotateAndMask - Returns true if Mask and Shift can be folded into a |
203 |
/// isRotateAndMask - Returns true if Mask and Shift can be folded into a |
| 204 |
/// rotate and mask opcode and mask operation. |
204 |
/// rotate and mask opcode and mask operation. |
| 205 |
static bool isRotateAndMask(SDNode *N, unsigned Mask, bool isShiftMask, |
205 |
static bool isRotateAndMask(SDNode *N, unsigned Mask, bool isShiftMask, |
| 206 |
unsigned &SH, unsigned &MB, unsigned &ME); |
206 |
unsigned &SH, unsigned &MB, unsigned &ME); |
| 207 |
|
207 |
|
| 208 |
/// getGlobalBaseReg - insert code into the entry mbb to materialize the PIC |
208 |
/// getGlobalBaseReg - insert code into the entry mbb to materialize the PIC |
| 209 |
/// base register. Return the virtual register that holds this value. |
209 |
/// base register. Return the virtual register that holds this value. |
| 210 |
SDNode *getGlobalBaseReg(); |
210 |
SDNode *getGlobalBaseReg(); |
| 211 |
|
211 |
|
| 212 |
void selectFrameIndex(SDNode *SN, SDNode *N, uint64_t Offset = 0); |
212 |
void selectFrameIndex(SDNode *SN, SDNode *N, uint64_t Offset = 0); |
| 213 |
|
213 |
|
| 214 |
// Select - Convert the specified operand from a target-independent to a |
214 |
// Select - Convert the specified operand from a target-independent to a |
| 215 |
// target-specific node if it hasn't already been changed. |
215 |
// target-specific node if it hasn't already been changed. |
| 216 |
void Select(SDNode *N) override; |
216 |
void Select(SDNode *N) override; |
| 217 |
|
217 |
|
| 218 |
bool tryBitfieldInsert(SDNode *N); |
218 |
bool tryBitfieldInsert(SDNode *N); |
| 219 |
bool tryBitPermutation(SDNode *N); |
219 |
bool tryBitPermutation(SDNode *N); |
| 220 |
bool tryIntCompareInGPR(SDNode *N); |
220 |
bool tryIntCompareInGPR(SDNode *N); |
| 221 |
|
221 |
|
| 222 |
// tryTLSXFormLoad - Convert an ISD::LOAD fed by a PPCISD::ADD_TLS into |
222 |
// tryTLSXFormLoad - Convert an ISD::LOAD fed by a PPCISD::ADD_TLS into |
| 223 |
// an X-Form load instruction with the offset being a relocation coming from |
223 |
// an X-Form load instruction with the offset being a relocation coming from |
| 224 |
// the PPCISD::ADD_TLS. |
224 |
// the PPCISD::ADD_TLS. |
| 225 |
bool tryTLSXFormLoad(LoadSDNode *N); |
225 |
bool tryTLSXFormLoad(LoadSDNode *N); |
| 226 |
// tryTLSXFormStore - Convert an ISD::STORE fed by a PPCISD::ADD_TLS into |
226 |
// tryTLSXFormStore - Convert an ISD::STORE fed by a PPCISD::ADD_TLS into |
| 227 |
// an X-Form store instruction with the offset being a relocation coming from |
227 |
// an X-Form store instruction with the offset being a relocation coming from |
| 228 |
// the PPCISD::ADD_TLS. |
228 |
// the PPCISD::ADD_TLS. |
| 229 |
bool tryTLSXFormStore(StoreSDNode *N); |
229 |
bool tryTLSXFormStore(StoreSDNode *N); |
| 230 |
/// SelectCC - Select a comparison of the specified values with the |
230 |
/// SelectCC - Select a comparison of the specified values with the |
| 231 |
/// specified condition code, returning the CR# of the expression. |
231 |
/// specified condition code, returning the CR# of the expression. |
| 232 |
SDValue SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
232 |
SDValue SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
| 233 |
const SDLoc &dl, SDValue Chain = SDValue()); |
233 |
const SDLoc &dl, SDValue Chain = SDValue()); |
| 234 |
|
234 |
|
| 235 |
/// SelectAddrImmOffs - Return true if the operand is valid for a preinc |
235 |
/// SelectAddrImmOffs - Return true if the operand is valid for a preinc |
| 236 |
/// immediate field. Note that the operand at this point is already the |
236 |
/// immediate field. Note that the operand at this point is already the |
| 237 |
/// result of a prior SelectAddressRegImm call. |
237 |
/// result of a prior SelectAddressRegImm call. |
| 238 |
bool SelectAddrImmOffs(SDValue N, SDValue &Out) const { |
238 |
bool SelectAddrImmOffs(SDValue N, SDValue &Out) const { |
| 239 |
if (N.getOpcode() == ISD::TargetConstant || |
239 |
if (N.getOpcode() == ISD::TargetConstant || |
| 240 |
N.getOpcode() == ISD::TargetGlobalAddress) { |
240 |
N.getOpcode() == ISD::TargetGlobalAddress) { |
| 241 |
Out = N; |
241 |
Out = N; |
| 242 |
return true; |
242 |
return true; |
| 243 |
} |
243 |
} |
| 244 |
|
244 |
|
| 245 |
return false; |
245 |
return false; |
| 246 |
} |
246 |
} |
| 247 |
|
247 |
|
| 248 |
/// SelectDSForm - Returns true if address N can be represented by the |
248 |
/// SelectDSForm - Returns true if address N can be represented by the |
| 249 |
/// addressing mode of DSForm instructions (a base register, plus a signed |
249 |
/// addressing mode of DSForm instructions (a base register, plus a signed |
| 250 |
/// 16-bit displacement that is a multiple of 4. |
250 |
/// 16-bit displacement that is a multiple of 4. |
| 251 |
bool SelectDSForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
251 |
bool SelectDSForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
| 252 |
return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, |
252 |
return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, |
| 253 |
Align(4)) == PPC::AM_DSForm; |
253 |
Align(4)) == PPC::AM_DSForm; |
| 254 |
} |
254 |
} |
| 255 |
|
255 |
|
| 256 |
/// SelectDQForm - Returns true if address N can be represented by the |
256 |
/// SelectDQForm - Returns true if address N can be represented by the |
| 257 |
/// addressing mode of DQForm instructions (a base register, plus a signed |
257 |
/// addressing mode of DQForm instructions (a base register, plus a signed |
| 258 |
/// 16-bit displacement that is a multiple of 16. |
258 |
/// 16-bit displacement that is a multiple of 16. |
| 259 |
bool SelectDQForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
259 |
bool SelectDQForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
| 260 |
return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, |
260 |
return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, |
| 261 |
Align(16)) == PPC::AM_DQForm; |
261 |
Align(16)) == PPC::AM_DQForm; |
| 262 |
} |
262 |
} |
| 263 |
|
263 |
|
| 264 |
/// SelectDForm - Returns true if address N can be represented by |
264 |
/// SelectDForm - Returns true if address N can be represented by |
| 265 |
/// the addressing mode of DForm instructions (a base register, plus a |
265 |
/// the addressing mode of DForm instructions (a base register, plus a |
| 266 |
/// signed 16-bit immediate. |
266 |
/// signed 16-bit immediate. |
| 267 |
bool SelectDForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
267 |
bool SelectDForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
| 268 |
return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, |
268 |
return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, |
| 269 |
std::nullopt) == PPC::AM_DForm; |
269 |
std::nullopt) == PPC::AM_DForm; |
| 270 |
} |
270 |
} |
| 271 |
|
271 |
|
| 272 |
/// SelectPCRelForm - Returns true if address N can be represented by |
272 |
/// SelectPCRelForm - Returns true if address N can be represented by |
| 273 |
/// PC-Relative addressing mode. |
273 |
/// PC-Relative addressing mode. |
| 274 |
bool SelectPCRelForm(SDNode *Parent, SDValue N, SDValue &Disp, |
274 |
bool SelectPCRelForm(SDNode *Parent, SDValue N, SDValue &Disp, |
| 275 |
SDValue &Base) { |
275 |
SDValue &Base) { |
| 276 |
return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, |
276 |
return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, |
| 277 |
std::nullopt) == PPC::AM_PCRel; |
277 |
std::nullopt) == PPC::AM_PCRel; |
| 278 |
} |
278 |
} |
| 279 |
|
279 |
|
| 280 |
/// SelectPDForm - Returns true if address N can be represented by Prefixed |
280 |
/// SelectPDForm - Returns true if address N can be represented by Prefixed |
| 281 |
/// DForm addressing mode (a base register, plus a signed 34-bit immediate. |
281 |
/// DForm addressing mode (a base register, plus a signed 34-bit immediate. |
| 282 |
bool SelectPDForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
282 |
bool SelectPDForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
| 283 |
return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, |
283 |
return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, |
| 284 |
std::nullopt) == |
284 |
std::nullopt) == |
| 285 |
PPC::AM_PrefixDForm; |
285 |
PPC::AM_PrefixDForm; |
| 286 |
} |
286 |
} |
| 287 |
|
287 |
|
| 288 |
/// SelectXForm - Returns true if address N can be represented by the |
288 |
/// SelectXForm - Returns true if address N can be represented by the |
| 289 |
/// addressing mode of XForm instructions (an indexed [r+r] operation). |
289 |
/// addressing mode of XForm instructions (an indexed [r+r] operation). |
| 290 |
bool SelectXForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
290 |
bool SelectXForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
| 291 |
return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, |
291 |
return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, |
| 292 |
std::nullopt) == PPC::AM_XForm; |
292 |
std::nullopt) == PPC::AM_XForm; |
| 293 |
} |
293 |
} |
| 294 |
|
294 |
|
| 295 |
/// SelectForceXForm - Given the specified address, force it to be |
295 |
/// SelectForceXForm - Given the specified address, force it to be |
| 296 |
/// represented as an indexed [r+r] operation (an XForm instruction). |
296 |
/// represented as an indexed [r+r] operation (an XForm instruction). |
| 297 |
bool SelectForceXForm(SDNode *Parent, SDValue N, SDValue &Disp, |
297 |
bool SelectForceXForm(SDNode *Parent, SDValue N, SDValue &Disp, |
| 298 |
SDValue &Base) { |
298 |
SDValue &Base) { |
| 299 |
return PPCLowering->SelectForceXFormMode(N, Disp, Base, *CurDAG) == |
299 |
return PPCLowering->SelectForceXFormMode(N, Disp, Base, *CurDAG) == |
| 300 |
PPC::AM_XForm; |
300 |
PPC::AM_XForm; |
| 301 |
} |
301 |
} |
| 302 |
|
302 |
|
| 303 |
/// SelectAddrIdx - Given the specified address, check to see if it can be |
303 |
/// SelectAddrIdx - Given the specified address, check to see if it can be |
| 304 |
/// represented as an indexed [r+r] operation. |
304 |
/// represented as an indexed [r+r] operation. |
| 305 |
/// This is for xform instructions whose associated displacement form is D. |
305 |
/// This is for xform instructions whose associated displacement form is D. |
| 306 |
/// The last parameter \p 0 means associated D form has no requirment for 16 |
306 |
/// The last parameter \p 0 means associated D form has no requirment for 16 |
| 307 |
/// bit signed displacement. |
307 |
/// bit signed displacement. |
| 308 |
/// Returns false if it can be represented by [r+imm], which are preferred. |
308 |
/// Returns false if it can be represented by [r+imm], which are preferred. |
| 309 |
bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) { |
309 |
bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) { |
| 310 |
return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, |
310 |
return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, |
| 311 |
std::nullopt); |
311 |
std::nullopt); |
| 312 |
} |
312 |
} |
| 313 |
|
313 |
|
| 314 |
/// SelectAddrIdx4 - Given the specified address, check to see if it can be |
314 |
/// SelectAddrIdx4 - Given the specified address, check to see if it can be |
| 315 |
/// represented as an indexed [r+r] operation. |
315 |
/// represented as an indexed [r+r] operation. |
| 316 |
/// This is for xform instructions whose associated displacement form is DS. |
316 |
/// This is for xform instructions whose associated displacement form is DS. |
| 317 |
/// The last parameter \p 4 means associated DS form 16 bit signed |
317 |
/// The last parameter \p 4 means associated DS form 16 bit signed |
| 318 |
/// displacement must be a multiple of 4. |
318 |
/// displacement must be a multiple of 4. |
| 319 |
/// Returns false if it can be represented by [r+imm], which are preferred. |
319 |
/// Returns false if it can be represented by [r+imm], which are preferred. |
| 320 |
bool SelectAddrIdxX4(SDValue N, SDValue &Base, SDValue &Index) { |
320 |
bool SelectAddrIdxX4(SDValue N, SDValue &Base, SDValue &Index) { |
| 321 |
return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, |
321 |
return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, |
| 322 |
Align(4)); |
322 |
Align(4)); |
| 323 |
} |
323 |
} |
| 324 |
|
324 |
|
| 325 |
/// SelectAddrIdx16 - Given the specified address, check to see if it can be |
325 |
/// SelectAddrIdx16 - Given the specified address, check to see if it can be |
| 326 |
/// represented as an indexed [r+r] operation. |
326 |
/// represented as an indexed [r+r] operation. |
| 327 |
/// This is for xform instructions whose associated displacement form is DQ. |
327 |
/// This is for xform instructions whose associated displacement form is DQ. |
| 328 |
/// The last parameter \p 16 means associated DQ form 16 bit signed |
328 |
/// The last parameter \p 16 means associated DQ form 16 bit signed |
| 329 |
/// displacement must be a multiple of 16. |
329 |
/// displacement must be a multiple of 16. |
| 330 |
/// Returns false if it can be represented by [r+imm], which are preferred. |
330 |
/// Returns false if it can be represented by [r+imm], which are preferred. |
| 331 |
bool SelectAddrIdxX16(SDValue N, SDValue &Base, SDValue &Index) { |
331 |
bool SelectAddrIdxX16(SDValue N, SDValue &Base, SDValue &Index) { |
| 332 |
return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, |
332 |
return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, |
| 333 |
Align(16)); |
333 |
Align(16)); |
| 334 |
} |
334 |
} |
| 335 |
|
335 |
|
| 336 |
/// SelectAddrIdxOnly - Given the specified address, force it to be |
336 |
/// SelectAddrIdxOnly - Given the specified address, force it to be |
| 337 |
/// represented as an indexed [r+r] operation. |
337 |
/// represented as an indexed [r+r] operation. |
| 338 |
bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) { |
338 |
bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) { |
| 339 |
return PPCLowering->SelectAddressRegRegOnly(N, Base, Index, *CurDAG); |
339 |
return PPCLowering->SelectAddressRegRegOnly(N, Base, Index, *CurDAG); |
| 340 |
} |
340 |
} |
| 341 |
|
341 |
|
| 342 |
/// SelectAddrImm - Returns true if the address N can be represented by |
342 |
/// SelectAddrImm - Returns true if the address N can be represented by |
| 343 |
/// a base register plus a signed 16-bit displacement [r+imm]. |
343 |
/// a base register plus a signed 16-bit displacement [r+imm]. |
| 344 |
/// The last parameter \p 0 means D form has no requirment for 16 bit signed |
344 |
/// The last parameter \p 0 means D form has no requirment for 16 bit signed |
| 345 |
/// displacement. |
345 |
/// displacement. |
| 346 |
bool SelectAddrImm(SDValue N, SDValue &Disp, |
346 |
bool SelectAddrImm(SDValue N, SDValue &Disp, |
| 347 |
SDValue &Base) { |
347 |
SDValue &Base) { |
| 348 |
return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, |
348 |
return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, |
| 349 |
std::nullopt); |
349 |
std::nullopt); |
| 350 |
} |
350 |
} |
| 351 |
|
351 |
|
| 352 |
/// SelectAddrImmX4 - Returns true if the address N can be represented by |
352 |
/// SelectAddrImmX4 - Returns true if the address N can be represented by |
| 353 |
/// a base register plus a signed 16-bit displacement that is a multiple of |
353 |
/// a base register plus a signed 16-bit displacement that is a multiple of |
| 354 |
/// 4 (last parameter). Suitable for use by STD and friends. |
354 |
/// 4 (last parameter). Suitable for use by STD and friends. |
| 355 |
bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) { |
355 |
bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) { |
| 356 |
return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, Align(4)); |
356 |
return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, Align(4)); |
| 357 |
} |
357 |
} |
| 358 |
|
358 |
|
| 359 |
/// SelectAddrImmX16 - Returns true if the address N can be represented by |
359 |
/// SelectAddrImmX16 - Returns true if the address N can be represented by |
| 360 |
/// a base register plus a signed 16-bit displacement that is a multiple of |
360 |
/// a base register plus a signed 16-bit displacement that is a multiple of |
| 361 |
/// 16(last parameter). Suitable for use by STXV and friends. |
361 |
/// 16(last parameter). Suitable for use by STXV and friends. |
| 362 |
bool SelectAddrImmX16(SDValue N, SDValue &Disp, SDValue &Base) { |
362 |
bool SelectAddrImmX16(SDValue N, SDValue &Disp, SDValue &Base) { |
| 363 |
return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, |
363 |
return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, |
| 364 |
Align(16)); |
364 |
Align(16)); |
| 365 |
} |
365 |
} |
| 366 |
|
366 |
|
| 367 |
/// SelectAddrImmX34 - Returns true if the address N can be represented by |
367 |
/// SelectAddrImmX34 - Returns true if the address N can be represented by |
| 368 |
/// a base register plus a signed 34-bit displacement. Suitable for use by |
368 |
/// a base register plus a signed 34-bit displacement. Suitable for use by |
| 369 |
/// PSTXVP and friends. |
369 |
/// PSTXVP and friends. |
| 370 |
bool SelectAddrImmX34(SDValue N, SDValue &Disp, SDValue &Base) { |
370 |
bool SelectAddrImmX34(SDValue N, SDValue &Disp, SDValue &Base) { |
| 371 |
return PPCLowering->SelectAddressRegImm34(N, Disp, Base, *CurDAG); |
371 |
return PPCLowering->SelectAddressRegImm34(N, Disp, Base, *CurDAG); |
| 372 |
} |
372 |
} |
| 373 |
|
373 |
|
| 374 |
// Select an address into a single register. |
374 |
// Select an address into a single register. |
| 375 |
bool SelectAddr(SDValue N, SDValue &Base) { |
375 |
bool SelectAddr(SDValue N, SDValue &Base) { |
| 376 |
Base = N; |
376 |
Base = N; |
| 377 |
return true; |
377 |
return true; |
| 378 |
} |
378 |
} |
| 379 |
|
379 |
|
| 380 |
bool SelectAddrPCRel(SDValue N, SDValue &Base) { |
380 |
bool SelectAddrPCRel(SDValue N, SDValue &Base) { |
| 381 |
return PPCLowering->SelectAddressPCRel(N, Base); |
381 |
return PPCLowering->SelectAddressPCRel(N, Base); |
| 382 |
} |
382 |
} |
| 383 |
|
383 |
|
| 384 |
/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for |
384 |
/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for |
| 385 |
/// inline asm expressions. It is always correct to compute the value into |
385 |
/// inline asm expressions. It is always correct to compute the value into |
| 386 |
/// a register. The case of adding a (possibly relocatable) constant to a |
386 |
/// a register. The case of adding a (possibly relocatable) constant to a |
| 387 |
/// register can be improved, but it is wrong to substitute Reg+Reg for |
387 |
/// register can be improved, but it is wrong to substitute Reg+Reg for |
| 388 |
/// Reg in an asm, because the load or store opcode would have to change. |
388 |
/// Reg in an asm, because the load or store opcode would have to change. |
| 389 |
bool SelectInlineAsmMemoryOperand(const SDValue &Op, |
389 |
bool SelectInlineAsmMemoryOperand(const SDValue &Op, |
| 390 |
unsigned ConstraintID, |
390 |
unsigned ConstraintID, |
| 391 |
std::vector &OutOps) override { |
391 |
std::vector &OutOps) override { |
| 392 |
switch(ConstraintID) { |
392 |
switch(ConstraintID) { |
| 393 |
default: |
393 |
default: |
| 394 |
errs() << "ConstraintID: " << ConstraintID << "\n"; |
394 |
errs() << "ConstraintID: " << ConstraintID << "\n"; |
| 395 |
llvm_unreachable("Unexpected asm memory constraint"); |
395 |
llvm_unreachable("Unexpected asm memory constraint"); |
| 396 |
case InlineAsm::Constraint_es: |
396 |
case InlineAsm::Constraint_es: |
| 397 |
case InlineAsm::Constraint_m: |
397 |
case InlineAsm::Constraint_m: |
| 398 |
case InlineAsm::Constraint_o: |
398 |
case InlineAsm::Constraint_o: |
| 399 |
case InlineAsm::Constraint_Q: |
399 |
case InlineAsm::Constraint_Q: |
| 400 |
case InlineAsm::Constraint_Z: |
400 |
case InlineAsm::Constraint_Z: |
| 401 |
case InlineAsm::Constraint_Zy: |
401 |
case InlineAsm::Constraint_Zy: |
| 402 |
// We need to make sure that this one operand does not end up in r0 |
402 |
// We need to make sure that this one operand does not end up in r0 |
| 403 |
// (because we might end up lowering this as 0(%op)). |
403 |
// (because we might end up lowering this as 0(%op)). |
| 404 |
const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo(); |
404 |
const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo(); |
| 405 |
const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF, /*Kind=*/1); |
405 |
const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF, /*Kind=*/1); |
| 406 |
SDLoc dl(Op); |
406 |
SDLoc dl(Op); |
| 407 |
SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32); |
407 |
SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32); |
| 408 |
SDValue NewOp = |
408 |
SDValue NewOp = |
| 409 |
SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, |
409 |
SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, |
| 410 |
dl, Op.getValueType(), |
410 |
dl, Op.getValueType(), |
| 411 |
Op, RC), 0); |
411 |
Op, RC), 0); |
| 412 |
|
412 |
|
| 413 |
OutOps.push_back(NewOp); |
413 |
OutOps.push_back(NewOp); |
| 414 |
return false; |
414 |
return false; |
| 415 |
} |
415 |
} |
| 416 |
return true; |
416 |
return true; |
| 417 |
} |
417 |
} |
| 418 |
|
418 |
|
| 419 |
// Include the pieces autogenerated from the target description. |
419 |
// Include the pieces autogenerated from the target description. |
| 420 |
#include "PPCGenDAGISel.inc" |
420 |
#include "PPCGenDAGISel.inc" |
| 421 |
|
421 |
|
| 422 |
private: |
422 |
private: |
| 423 |
bool trySETCC(SDNode *N); |
423 |
bool trySETCC(SDNode *N); |
| 424 |
bool tryFoldSWTestBRCC(SDNode *N); |
424 |
bool tryFoldSWTestBRCC(SDNode *N); |
| 425 |
bool trySelectLoopCountIntrinsic(SDNode *N); |
425 |
bool trySelectLoopCountIntrinsic(SDNode *N); |
| 426 |
bool tryAsSingleRLDICL(SDNode *N); |
426 |
bool tryAsSingleRLDICL(SDNode *N); |
| 427 |
bool tryAsSingleRLDICR(SDNode *N); |
427 |
bool tryAsSingleRLDICR(SDNode *N); |
| 428 |
bool tryAsSingleRLWINM(SDNode *N); |
428 |
bool tryAsSingleRLWINM(SDNode *N); |
| 429 |
bool tryAsSingleRLWINM8(SDNode *N); |
429 |
bool tryAsSingleRLWINM8(SDNode *N); |
| 430 |
bool tryAsSingleRLWIMI(SDNode *N); |
430 |
bool tryAsSingleRLWIMI(SDNode *N); |
| 431 |
bool tryAsPairOfRLDICL(SDNode *N); |
431 |
bool tryAsPairOfRLDICL(SDNode *N); |
| 432 |
bool tryAsSingleRLDIMI(SDNode *N); |
432 |
bool tryAsSingleRLDIMI(SDNode *N); |
| 433 |
|
433 |
|
| 434 |
void PeepholePPC64(); |
434 |
void PeepholePPC64(); |
| 435 |
void PeepholePPC64ZExt(); |
435 |
void PeepholePPC64ZExt(); |
| 436 |
void PeepholeCROps(); |
436 |
void PeepholeCROps(); |
| 437 |
|
437 |
|
| 438 |
SDValue combineToCMPB(SDNode *N); |
438 |
SDValue combineToCMPB(SDNode *N); |
| 439 |
void foldBoolExts(SDValue &Res, SDNode *&N); |
439 |
void foldBoolExts(SDValue &Res, SDNode *&N); |
| 440 |
|
440 |
|
| 441 |
bool AllUsersSelectZero(SDNode *N); |
441 |
bool AllUsersSelectZero(SDNode *N); |
| 442 |
void SwapAllSelectUsers(SDNode *N); |
442 |
void SwapAllSelectUsers(SDNode *N); |
| 443 |
|
443 |
|
| 444 |
bool isOffsetMultipleOf(SDNode *N, unsigned Val) const; |
444 |
bool isOffsetMultipleOf(SDNode *N, unsigned Val) const; |
| 445 |
void transferMemOperands(SDNode *N, SDNode *Result); |
445 |
void transferMemOperands(SDNode *N, SDNode *Result); |
| 446 |
}; |
446 |
}; |
| 447 |
|
447 |
|
| 448 |
} // end anonymous namespace |
448 |
} // end anonymous namespace |
| 449 |
|
449 |
|
| 450 |
char PPCDAGToDAGISel::ID = 0; |
450 |
char PPCDAGToDAGISel::ID = 0; |
| 451 |
|
451 |
|
| 452 |
INITIALIZE_PASS(PPCDAGToDAGISel, DEBUG_TYPE, PASS_NAME, false, false) |
452 |
INITIALIZE_PASS(PPCDAGToDAGISel, DEBUG_TYPE, PASS_NAME, false, false) |
| 453 |
|
453 |
|
| 454 |
/// getGlobalBaseReg - Output the instructions required to put the |
454 |
/// getGlobalBaseReg - Output the instructions required to put the |
| 455 |
/// base address to use for accessing globals into a register. |
455 |
/// base address to use for accessing globals into a register. |
| 456 |
/// |
456 |
/// |
| 457 |
SDNode *PPCDAGToDAGISel::getGlobalBaseReg() { |
457 |
SDNode *PPCDAGToDAGISel::getGlobalBaseReg() { |
| 458 |
if (!GlobalBaseReg) { |
458 |
if (!GlobalBaseReg) { |
| 459 |
const TargetInstrInfo &TII = *Subtarget->getInstrInfo(); |
459 |
const TargetInstrInfo &TII = *Subtarget->getInstrInfo(); |
| 460 |
// Insert the set of GlobalBaseReg into the first MBB of the function |
460 |
// Insert the set of GlobalBaseReg into the first MBB of the function |
| 461 |
MachineBasicBlock &FirstMBB = MF->front(); |
461 |
MachineBasicBlock &FirstMBB = MF->front(); |
| 462 |
MachineBasicBlock::iterator MBBI = FirstMBB.begin(); |
462 |
MachineBasicBlock::iterator MBBI = FirstMBB.begin(); |
| 463 |
const Module *M = MF->getFunction().getParent(); |
463 |
const Module *M = MF->getFunction().getParent(); |
| 464 |
DebugLoc dl; |
464 |
DebugLoc dl; |
| 465 |
|
465 |
|
| 466 |
if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) == MVT::i32) { |
466 |
if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) == MVT::i32) { |
| 467 |
if (Subtarget->isTargetELF()) { |
467 |
if (Subtarget->isTargetELF()) { |
| 468 |
GlobalBaseReg = PPC::R30; |
468 |
GlobalBaseReg = PPC::R30; |
| 469 |
if (!Subtarget->isSecurePlt() && |
469 |
if (!Subtarget->isSecurePlt() && |
| 470 |
M->getPICLevel() == PICLevel::SmallPIC) { |
470 |
M->getPICLevel() == PICLevel::SmallPIC) { |
| 471 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MoveGOTtoLR)); |
471 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MoveGOTtoLR)); |
| 472 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); |
472 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); |
| 473 |
MF->getInfo()->setUsesPICBase(true); |
473 |
MF->getInfo()->setUsesPICBase(true); |
| 474 |
} else { |
474 |
} else { |
| 475 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR)); |
475 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR)); |
| 476 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); |
476 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); |
| 477 |
Register TempReg = RegInfo->createVirtualRegister(&PPC::GPRCRegClass); |
477 |
Register TempReg = RegInfo->createVirtualRegister(&PPC::GPRCRegClass); |
| 478 |
BuildMI(FirstMBB, MBBI, dl, |
478 |
BuildMI(FirstMBB, MBBI, dl, |
| 479 |
TII.get(PPC::UpdateGBR), GlobalBaseReg) |
479 |
TII.get(PPC::UpdateGBR), GlobalBaseReg) |
| 480 |
.addReg(TempReg, RegState::Define).addReg(GlobalBaseReg); |
480 |
.addReg(TempReg, RegState::Define).addReg(GlobalBaseReg); |
| 481 |
MF->getInfo()->setUsesPICBase(true); |
481 |
MF->getInfo()->setUsesPICBase(true); |
| 482 |
} |
482 |
} |
| 483 |
} else { |
483 |
} else { |
| 484 |
GlobalBaseReg = |
484 |
GlobalBaseReg = |
| 485 |
RegInfo->createVirtualRegister(&PPC::GPRC_and_GPRC_NOR0RegClass); |
485 |
RegInfo->createVirtualRegister(&PPC::GPRC_and_GPRC_NOR0RegClass); |
| 486 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR)); |
486 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR)); |
| 487 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); |
487 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); |
| 488 |
} |
488 |
} |
| 489 |
} else { |
489 |
} else { |
| 490 |
// We must ensure that this sequence is dominated by the prologue. |
490 |
// We must ensure that this sequence is dominated by the prologue. |
| 491 |
// FIXME: This is a bit of a big hammer since we don't get the benefits |
491 |
// FIXME: This is a bit of a big hammer since we don't get the benefits |
| 492 |
// of shrink-wrapping whenever we emit this instruction. Considering |
492 |
// of shrink-wrapping whenever we emit this instruction. Considering |
| 493 |
// this is used in any function where we emit a jump table, this may be |
493 |
// this is used in any function where we emit a jump table, this may be |
| 494 |
// a significant limitation. We should consider inserting this in the |
494 |
// a significant limitation. We should consider inserting this in the |
| 495 |
// block where it is used and then commoning this sequence up if it |
495 |
// block where it is used and then commoning this sequence up if it |
| 496 |
// appears in multiple places. |
496 |
// appears in multiple places. |
| 497 |
// Note: on ISA 3.0 cores, we can use lnia (addpcis) instead of |
497 |
// Note: on ISA 3.0 cores, we can use lnia (addpcis) instead of |
| 498 |
// MovePCtoLR8. |
498 |
// MovePCtoLR8. |
| 499 |
MF->getInfo()->setShrinkWrapDisabled(true); |
499 |
MF->getInfo()->setShrinkWrapDisabled(true); |
| 500 |
GlobalBaseReg = RegInfo->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass); |
500 |
GlobalBaseReg = RegInfo->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass); |
| 501 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR8)); |
501 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR8)); |
| 502 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR8), GlobalBaseReg); |
502 |
BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR8), GlobalBaseReg); |
| 503 |
} |
503 |
} |
| 504 |
} |
504 |
} |
| 505 |
return CurDAG->getRegister(GlobalBaseReg, |
505 |
return CurDAG->getRegister(GlobalBaseReg, |
| 506 |
PPCLowering->getPointerTy(CurDAG->getDataLayout())) |
506 |
PPCLowering->getPointerTy(CurDAG->getDataLayout())) |
| 507 |
.getNode(); |
507 |
.getNode(); |
| 508 |
} |
508 |
} |
| 509 |
|
509 |
|
| 510 |
// Check if a SDValue has the toc-data attribute. |
510 |
// Check if a SDValue has the toc-data attribute. |
| 511 |
static bool hasTocDataAttr(SDValue Val, unsigned PointerSize) { |
511 |
static bool hasTocDataAttr(SDValue Val, unsigned PointerSize) { |
| 512 |
GlobalAddressSDNode *GA = dyn_cast(Val); |
512 |
GlobalAddressSDNode *GA = dyn_cast(Val); |
| 513 |
if (!GA) |
513 |
if (!GA) |
| 514 |
return false; |
514 |
return false; |
| 515 |
|
515 |
|
| 516 |
const GlobalVariable *GV = dyn_cast_or_null(GA->getGlobal()); |
516 |
const GlobalVariable *GV = dyn_cast_or_null(GA->getGlobal()); |
| 517 |
if (!GV) |
517 |
if (!GV) |
| 518 |
return false; |
518 |
return false; |
| 519 |
|
519 |
|
| 520 |
if (!GV->hasAttribute("toc-data")) |
520 |
if (!GV->hasAttribute("toc-data")) |
| 521 |
return false; |
521 |
return false; |
| 522 |
|
522 |
|
| 523 |
// TODO: These asserts should be updated as more support for the toc data |
523 |
// TODO: These asserts should be updated as more support for the toc data |
| 524 |
// transformation is added (struct support, etc.). |
524 |
// transformation is added (struct support, etc.). |
| 525 |
|
525 |
|
| 526 |
assert( |
526 |
assert( |
| 527 |
PointerSize >= GV->getAlign().valueOrOne().value() && |
527 |
PointerSize >= GV->getAlign().valueOrOne().value() && |
| 528 |
"GlobalVariables with an alignment requirement stricter than TOC entry " |
528 |
"GlobalVariables with an alignment requirement stricter than TOC entry " |
| 529 |
"size not supported by the toc data transformation."); |
529 |
"size not supported by the toc data transformation."); |
| 530 |
|
530 |
|
| 531 |
Type *GVType = GV->getValueType(); |
531 |
Type *GVType = GV->getValueType(); |
| 532 |
|
532 |
|
| 533 |
assert(GVType->isSized() && "A GlobalVariable's size must be known to be " |
533 |
assert(GVType->isSized() && "A GlobalVariable's size must be known to be " |
| 534 |
"supported by the toc data transformation."); |
534 |
"supported by the toc data transformation."); |
| 535 |
|
535 |
|
| 536 |
if (GVType->isVectorTy()) |
536 |
if (GVType->isVectorTy()) |
| 537 |
report_fatal_error("A GlobalVariable of Vector type is not currently " |
537 |
report_fatal_error("A GlobalVariable of Vector type is not currently " |
| 538 |
"supported by the toc data transformation."); |
538 |
"supported by the toc data transformation."); |
| 539 |
|
539 |
|
| 540 |
if (GVType->isArrayTy()) |
540 |
if (GVType->isArrayTy()) |
| 541 |
report_fatal_error("A GlobalVariable of Array type is not currently " |
541 |
report_fatal_error("A GlobalVariable of Array type is not currently " |
| 542 |
"supported by the toc data transformation."); |
542 |
"supported by the toc data transformation."); |
| 543 |
|
543 |
|
| 544 |
if (GVType->isStructTy()) |
544 |
if (GVType->isStructTy()) |
| 545 |
report_fatal_error("A GlobalVariable of Struct type is not currently " |
545 |
report_fatal_error("A GlobalVariable of Struct type is not currently " |
| 546 |
"supported by the toc data transformation."); |
546 |
"supported by the toc data transformation."); |
| 547 |
|
547 |
|
| 548 |
assert(GVType->getPrimitiveSizeInBits() <= PointerSize * 8 && |
548 |
assert(GVType->getPrimitiveSizeInBits() <= PointerSize * 8 && |
| 549 |
"A GlobalVariable with size larger than a TOC entry is not currently " |
549 |
"A GlobalVariable with size larger than a TOC entry is not currently " |
| 550 |
"supported by the toc data transformation."); |
550 |
"supported by the toc data transformation."); |
| 551 |
|
551 |
|
| 552 |
if (GV->hasLocalLinkage() || GV->hasPrivateLinkage()) |
552 |
if (GV->hasLocalLinkage() || GV->hasPrivateLinkage()) |
| 553 |
report_fatal_error("A GlobalVariable with private or local linkage is not " |
553 |
report_fatal_error("A GlobalVariable with private or local linkage is not " |
| 554 |
"currently supported by the toc data transformation."); |
554 |
"currently supported by the toc data transformation."); |
| 555 |
|
555 |
|
| 556 |
assert(!GV->hasCommonLinkage() && |
556 |
assert(!GV->hasCommonLinkage() && |
| 557 |
"Tentative definitions cannot have the mapping class XMC_TD."); |
557 |
"Tentative definitions cannot have the mapping class XMC_TD."); |
| 558 |
|
558 |
|
| 559 |
return true; |
559 |
return true; |
| 560 |
} |
560 |
} |
| 561 |
|
561 |
|
| 562 |
/// isInt32Immediate - This method tests to see if the node is a 32-bit constant |
562 |
/// isInt32Immediate - This method tests to see if the node is a 32-bit constant |
| 563 |
/// operand. If so Imm will receive the 32-bit value. |
563 |
/// operand. If so Imm will receive the 32-bit value. |
| 564 |
static bool isInt32Immediate(SDNode *N, unsigned &Imm) { |
564 |
static bool isInt32Immediate(SDNode *N, unsigned &Imm) { |
| 565 |
if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { |
565 |
if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { |
| 566 |
Imm = cast(N)->getZExtValue(); |
566 |
Imm = cast(N)->getZExtValue(); |
| 567 |
return true; |
567 |
return true; |
| 568 |
} |
568 |
} |
| 569 |
return false; |
569 |
return false; |
| 570 |
} |
570 |
} |
| 571 |
|
571 |
|
| 572 |
/// isInt64Immediate - This method tests to see if the node is a 64-bit constant |
572 |
/// isInt64Immediate - This method tests to see if the node is a 64-bit constant |
| 573 |
/// operand. If so Imm will receive the 64-bit value. |
573 |
/// operand. If so Imm will receive the 64-bit value. |
| 574 |
static bool isInt64Immediate(SDNode *N, uint64_t &Imm) { |
574 |
static bool isInt64Immediate(SDNode *N, uint64_t &Imm) { |
| 575 |
if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i64) { |
575 |
if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i64) { |
| 576 |
Imm = cast(N)->getZExtValue(); |
576 |
Imm = cast(N)->getZExtValue(); |
| 577 |
return true; |
577 |
return true; |
| 578 |
} |
578 |
} |
| 579 |
return false; |
579 |
return false; |
| 580 |
} |
580 |
} |
| 581 |
|
581 |
|
| 582 |
// isInt32Immediate - This method tests to see if a constant operand. |
582 |
// isInt32Immediate - This method tests to see if a constant operand. |
| 583 |
// If so Imm will receive the 32 bit value. |
583 |
// If so Imm will receive the 32 bit value. |
| 584 |
static bool isInt32Immediate(SDValue N, unsigned &Imm) { |
584 |
static bool isInt32Immediate(SDValue N, unsigned &Imm) { |
| 585 |
return isInt32Immediate(N.getNode(), Imm); |
585 |
return isInt32Immediate(N.getNode(), Imm); |
| 586 |
} |
586 |
} |
| 587 |
|
587 |
|
| 588 |
/// isInt64Immediate - This method tests to see if the value is a 64-bit |
588 |
/// isInt64Immediate - This method tests to see if the value is a 64-bit |
| 589 |
/// constant operand. If so Imm will receive the 64-bit value. |
589 |
/// constant operand. If so Imm will receive the 64-bit value. |
| 590 |
static bool isInt64Immediate(SDValue N, uint64_t &Imm) { |
590 |
static bool isInt64Immediate(SDValue N, uint64_t &Imm) { |
| 591 |
return isInt64Immediate(N.getNode(), Imm); |
591 |
return isInt64Immediate(N.getNode(), Imm); |
| 592 |
} |
592 |
} |
| 593 |
|
593 |
|
| 594 |
static unsigned getBranchHint(unsigned PCC, |
594 |
static unsigned getBranchHint(unsigned PCC, |
| 595 |
const FunctionLoweringInfo &FuncInfo, |
595 |
const FunctionLoweringInfo &FuncInfo, |
| 596 |
const SDValue &DestMBB) { |
596 |
const SDValue &DestMBB) { |
| 597 |
assert(isa(DestMBB)); |
597 |
assert(isa(DestMBB)); |
| 598 |
|
598 |
|
| 599 |
if (!FuncInfo.BPI) return PPC::BR_NO_HINT; |
599 |
if (!FuncInfo.BPI) return PPC::BR_NO_HINT; |
| 600 |
|
600 |
|
| 601 |
const BasicBlock *BB = FuncInfo.MBB->getBasicBlock(); |
601 |
const BasicBlock *BB = FuncInfo.MBB->getBasicBlock(); |
| 602 |
const Instruction *BBTerm = BB->getTerminator(); |
602 |
const Instruction *BBTerm = BB->getTerminator(); |
| 603 |
|
603 |
|
| 604 |
if (BBTerm->getNumSuccessors() != 2) return PPC::BR_NO_HINT; |
604 |
if (BBTerm->getNumSuccessors() != 2) return PPC::BR_NO_HINT; |
| 605 |
|
605 |
|
| 606 |
const BasicBlock *TBB = BBTerm->getSuccessor(0); |
606 |
const BasicBlock *TBB = BBTerm->getSuccessor(0); |
| 607 |
const BasicBlock *FBB = BBTerm->getSuccessor(1); |
607 |
const BasicBlock *FBB = BBTerm->getSuccessor(1); |
| 608 |
|
608 |
|
| 609 |
auto TProb = FuncInfo.BPI->getEdgeProbability(BB, TBB); |
609 |
auto TProb = FuncInfo.BPI->getEdgeProbability(BB, TBB); |
| 610 |
auto FProb = FuncInfo.BPI->getEdgeProbability(BB, FBB); |
610 |
auto FProb = FuncInfo.BPI->getEdgeProbability(BB, FBB); |
| 611 |
|
611 |
|
| 612 |
// We only want to handle cases which are easy to predict at static time, e.g. |
612 |
// We only want to handle cases which are easy to predict at static time, e.g. |
| 613 |
// C++ throw statement, that is very likely not taken, or calling never |
613 |
// C++ throw statement, that is very likely not taken, or calling never |
| 614 |
// returned function, e.g. stdlib exit(). So we set Threshold to filter |
614 |
// returned function, e.g. stdlib exit(). So we set Threshold to filter |
| 615 |
// unwanted cases. |
615 |
// unwanted cases. |
| 616 |
// |
616 |
// |
| 617 |
// Below is LLVM branch weight table, we only want to handle case 1, 2 |
617 |
// Below is LLVM branch weight table, we only want to handle case 1, 2 |
| 618 |
// |
618 |
// |
| 619 |
// Case Taken:Nontaken Example |
619 |
// Case Taken:Nontaken Example |
| 620 |
// 1. Unreachable 1048575:1 C++ throw, stdlib exit(), |
620 |
// 1. Unreachable 1048575:1 C++ throw, stdlib exit(), |
| 621 |
// 2. Invoke-terminating 1:1048575 |
621 |
// 2. Invoke-terminating 1:1048575 |
| 622 |
// 3. Coldblock 4:64 __builtin_expect |
622 |
// 3. Coldblock 4:64 __builtin_expect |
| 623 |
// 4. Loop Branch 124:4 For loop |
623 |
// 4. Loop Branch 124:4 For loop |
| 624 |
// 5. PH/ZH/FPH 20:12 |
624 |
// 5. PH/ZH/FPH 20:12 |
| 625 |
const uint32_t Threshold = 10000; |
625 |
const uint32_t Threshold = 10000; |
| 626 |
|
626 |
|
| 627 |
if (std::max(TProb, FProb) / Threshold < std::min(TProb, FProb)) |
627 |
if (std::max(TProb, FProb) / Threshold < std::min(TProb, FProb)) |
| 628 |
return PPC::BR_NO_HINT; |
628 |
return PPC::BR_NO_HINT; |
| 629 |
|
629 |
|
| 630 |
LLVM_DEBUG(dbgs() << "Use branch hint for '" << FuncInfo.Fn->getName() |
630 |
LLVM_DEBUG(dbgs() << "Use branch hint for '" << FuncInfo.Fn->getName() |
| 631 |
<< "::" << BB->getName() << "'\n" |
631 |
<< "::" << BB->getName() << "'\n" |
| 632 |
<< " -> " << TBB->getName() << ": " << TProb << "\n" |
632 |
<< " -> " << TBB->getName() << ": " << TProb << "\n" |
| 633 |
<< " -> " << FBB->getName() << ": " << FProb << "\n"); |
633 |
<< " -> " << FBB->getName() << ": " << FProb << "\n"); |
| 634 |
|
634 |
|
| 635 |
const BasicBlockSDNode *BBDN = cast(DestMBB); |
635 |
const BasicBlockSDNode *BBDN = cast(DestMBB); |
| 636 |
|
636 |
|
| 637 |
// If Dest BasicBlock is False-BasicBlock (FBB), swap branch probabilities, |
637 |
// If Dest BasicBlock is False-BasicBlock (FBB), swap branch probabilities, |
| 638 |
// because we want 'TProb' stands for 'branch probability' to Dest BasicBlock |
638 |
// because we want 'TProb' stands for 'branch probability' to Dest BasicBlock |
| 639 |
if (BBDN->getBasicBlock()->getBasicBlock() != TBB) |
639 |
if (BBDN->getBasicBlock()->getBasicBlock() != TBB) |
| 640 |
std::swap(TProb, FProb); |
640 |
std::swap(TProb, FProb); |
| 641 |
|
641 |
|
| 642 |
return (TProb > FProb) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT; |
642 |
return (TProb > FProb) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT; |
| 643 |
} |
643 |
} |
| 644 |
|
644 |
|
| 645 |
// isOpcWithIntImmediate - This method tests to see if the node is a specific |
645 |
// isOpcWithIntImmediate - This method tests to see if the node is a specific |
| 646 |
// opcode and that it has a immediate integer right operand. |
646 |
// opcode and that it has a immediate integer right operand. |
| 647 |
// If so Imm will receive the 32 bit value. |
647 |
// If so Imm will receive the 32 bit value. |
| 648 |
static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) { |
648 |
static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) { |
| 649 |
return N->getOpcode() == Opc |
649 |
return N->getOpcode() == Opc |
| 650 |
&& isInt32Immediate(N->getOperand(1).getNode(), Imm); |
650 |
&& isInt32Immediate(N->getOperand(1).getNode(), Imm); |
| 651 |
} |
651 |
} |
| 652 |
|
652 |
|
| 653 |
void PPCDAGToDAGISel::selectFrameIndex(SDNode *SN, SDNode *N, uint64_t Offset) { |
653 |
void PPCDAGToDAGISel::selectFrameIndex(SDNode *SN, SDNode *N, uint64_t Offset) { |
| 654 |
SDLoc dl(SN); |
654 |
SDLoc dl(SN); |
| 655 |
int FI = cast(N)->getIndex(); |
655 |
int FI = cast(N)->getIndex(); |
| 656 |
SDValue TFI = CurDAG->getTargetFrameIndex(FI, N->getValueType(0)); |
656 |
SDValue TFI = CurDAG->getTargetFrameIndex(FI, N->getValueType(0)); |
| 657 |
unsigned Opc = N->getValueType(0) == MVT::i32 ? PPC::ADDI : PPC::ADDI8; |
657 |
unsigned Opc = N->getValueType(0) == MVT::i32 ? PPC::ADDI : PPC::ADDI8; |
| 658 |
if (SN->hasOneUse()) |
658 |
if (SN->hasOneUse()) |
| 659 |
CurDAG->SelectNodeTo(SN, Opc, N->getValueType(0), TFI, |
659 |
CurDAG->SelectNodeTo(SN, Opc, N->getValueType(0), TFI, |
| 660 |
getSmallIPtrImm(Offset, dl)); |
660 |
getSmallIPtrImm(Offset, dl)); |
| 661 |
else |
661 |
else |
| 662 |
ReplaceNode(SN, CurDAG->getMachineNode(Opc, dl, N->getValueType(0), TFI, |
662 |
ReplaceNode(SN, CurDAG->getMachineNode(Opc, dl, N->getValueType(0), TFI, |
| 663 |
getSmallIPtrImm(Offset, dl))); |
663 |
getSmallIPtrImm(Offset, dl))); |
| 664 |
} |
664 |
} |
| 665 |
|
665 |
|
| 666 |
bool PPCDAGToDAGISel::isRotateAndMask(SDNode *N, unsigned Mask, |
666 |
bool PPCDAGToDAGISel::isRotateAndMask(SDNode *N, unsigned Mask, |
| 667 |
bool isShiftMask, unsigned &SH, |
667 |
bool isShiftMask, unsigned &SH, |
| 668 |
unsigned &MB, unsigned &ME) { |
668 |
unsigned &MB, unsigned &ME) { |
| 669 |
// Don't even go down this path for i64, since different logic will be |
669 |
// Don't even go down this path for i64, since different logic will be |
| 670 |
// necessary for rldicl/rldicr/rldimi. |
670 |
// necessary for rldicl/rldicr/rldimi. |
| 671 |
if (N->getValueType(0) != MVT::i32) |
671 |
if (N->getValueType(0) != MVT::i32) |
| 672 |
return false; |
672 |
return false; |
| 673 |
|
673 |
|
| 674 |
unsigned Shift = 32; |
674 |
unsigned Shift = 32; |
| 675 |
unsigned Indeterminant = ~0; // bit mask marking indeterminant results |
675 |
unsigned Indeterminant = ~0; // bit mask marking indeterminant results |
| 676 |
unsigned Opcode = N->getOpcode(); |
676 |
unsigned Opcode = N->getOpcode(); |
| 677 |
if (N->getNumOperands() != 2 || |
677 |
if (N->getNumOperands() != 2 || |
| 678 |
!isInt32Immediate(N->getOperand(1).getNode(), Shift) || (Shift > 31)) |
678 |
!isInt32Immediate(N->getOperand(1).getNode(), Shift) || (Shift > 31)) |
| 679 |
return false; |
679 |
return false; |
| 680 |
|
680 |
|
| 681 |
if (Opcode == ISD::SHL) { |
681 |
if (Opcode == ISD::SHL) { |
| 682 |
// apply shift left to mask if it comes first |
682 |
// apply shift left to mask if it comes first |
| 683 |
if (isShiftMask) Mask = Mask << Shift; |
683 |
if (isShiftMask) Mask = Mask << Shift; |
| 684 |
// determine which bits are made indeterminant by shift |
684 |
// determine which bits are made indeterminant by shift |
| 685 |
Indeterminant = ~(0xFFFFFFFFu << Shift); |
685 |
Indeterminant = ~(0xFFFFFFFFu << Shift); |
| 686 |
} else if (Opcode == ISD::SRL) { |
686 |
} else if (Opcode == ISD::SRL) { |
| 687 |
// apply shift right to mask if it comes first |
687 |
// apply shift right to mask if it comes first |
| 688 |
if (isShiftMask) Mask = Mask >> Shift; |
688 |
if (isShiftMask) Mask = Mask >> Shift; |
| 689 |
// determine which bits are made indeterminant by shift |
689 |
// determine which bits are made indeterminant by shift |
| 690 |
Indeterminant = ~(0xFFFFFFFFu >> Shift); |
690 |
Indeterminant = ~(0xFFFFFFFFu >> Shift); |
| 691 |
// adjust for the left rotate |
691 |
// adjust for the left rotate |
| 692 |
Shift = 32 - Shift; |
692 |
Shift = 32 - Shift; |
| 693 |
} else if (Opcode == ISD::ROTL) { |
693 |
} else if (Opcode == ISD::ROTL) { |
| 694 |
Indeterminant = 0; |
694 |
Indeterminant = 0; |
| 695 |
} else { |
695 |
} else { |
| 696 |
return false; |
696 |
return false; |
| 697 |
} |
697 |
} |
| 698 |
|
698 |
|
| 699 |
// if the mask doesn't intersect any Indeterminant bits |
699 |
// if the mask doesn't intersect any Indeterminant bits |
| 700 |
if (Mask && !(Mask & Indeterminant)) { |
700 |
if (Mask && !(Mask & Indeterminant)) { |
| 701 |
SH = Shift & 31; |
701 |
SH = Shift & 31; |
| 702 |
// make sure the mask is still a mask (wrap arounds may not be) |
702 |
// make sure the mask is still a mask (wrap arounds may not be) |
| 703 |
return isRunOfOnes(Mask, MB, ME); |
703 |
return isRunOfOnes(Mask, MB, ME); |
| 704 |
} |
704 |
} |
| 705 |
return false; |
705 |
return false; |
| 706 |
} |
706 |
} |
| 707 |
|
707 |
|
| 708 |
// isThreadPointerAcquisitionNode - Check if the operands of an ADD_TLS |
708 |
// isThreadPointerAcquisitionNode - Check if the operands of an ADD_TLS |
| 709 |
// instruction use the thread pointer. |
709 |
// instruction use the thread pointer. |
| 710 |
static bool isThreadPointerAcquisitionNode(SDValue Base, SelectionDAG *CurDAG) { |
710 |
static bool isThreadPointerAcquisitionNode(SDValue Base, SelectionDAG *CurDAG) { |
| 711 |
assert( |
711 |
assert( |
| 712 |
Base.getOpcode() == PPCISD::ADD_TLS && |
712 |
Base.getOpcode() == PPCISD::ADD_TLS && |
| 713 |
"Only expecting the ADD_TLS instruction to acquire the thread pointer!"); |
713 |
"Only expecting the ADD_TLS instruction to acquire the thread pointer!"); |
| 714 |
const PPCSubtarget &Subtarget = |
714 |
const PPCSubtarget &Subtarget = |
| 715 |
CurDAG->getMachineFunction().getSubtarget(); |
715 |
CurDAG->getMachineFunction().getSubtarget(); |
| 716 |
SDValue ADDTLSOp1 = Base.getOperand(0); |
716 |
SDValue ADDTLSOp1 = Base.getOperand(0); |
| 717 |
unsigned ADDTLSOp1Opcode = ADDTLSOp1.getOpcode(); |
717 |
unsigned ADDTLSOp1Opcode = ADDTLSOp1.getOpcode(); |
| 718 |
|
718 |
|
| 719 |
// Account for when ADD_TLS is used for the initial-exec TLS model on Linux. |
719 |
// Account for when ADD_TLS is used for the initial-exec TLS model on Linux. |
| 720 |
// |
720 |
// |
| 721 |
// Although ADD_TLS does not explicitly use the thread pointer |
721 |
// Although ADD_TLS does not explicitly use the thread pointer |
| 722 |
// register when LD_GOT_TPREL_L is one of it's operands, the LD_GOT_TPREL_L |
722 |
// register when LD_GOT_TPREL_L is one of it's operands, the LD_GOT_TPREL_L |
| 723 |
// instruction will have a relocation specifier, @got@tprel, that is used to |
723 |
// instruction will have a relocation specifier, @got@tprel, that is used to |
| 724 |
// generate a GOT entry. The linker replaces this entry with an offset for a |
724 |
// generate a GOT entry. The linker replaces this entry with an offset for a |
| 725 |
// for a thread local variable, which will be relative to the thread pointer. |
725 |
// for a thread local variable, which will be relative to the thread pointer. |
| 726 |
if (ADDTLSOp1Opcode == PPCISD::LD_GOT_TPREL_L) |
726 |
if (ADDTLSOp1Opcode == PPCISD::LD_GOT_TPREL_L) |
| 727 |
return true; |
727 |
return true; |
| 728 |
// When using PC-Relative instructions for initial-exec, a MAT_PCREL_ADDR |
728 |
// When using PC-Relative instructions for initial-exec, a MAT_PCREL_ADDR |
| 729 |
// node is produced instead to represent the aforementioned situation. |
729 |
// node is produced instead to represent the aforementioned situation. |
| 730 |
LoadSDNode *LD = dyn_cast(ADDTLSOp1); |
730 |
LoadSDNode *LD = dyn_cast(ADDTLSOp1); |
| 731 |
if (LD && LD->getBasePtr().getOpcode() == PPCISD::MAT_PCREL_ADDR) |
731 |
if (LD && LD->getBasePtr().getOpcode() == PPCISD::MAT_PCREL_ADDR) |
| 732 |
return true; |
732 |
return true; |
| 733 |
|
733 |
|
| 734 |
// A GET_TPOINTER PPCISD node (only produced on AIX 32-bit mode) as an operand |
734 |
// A GET_TPOINTER PPCISD node (only produced on AIX 32-bit mode) as an operand |
| 735 |
// to ADD_TLS represents a call to .__get_tpointer to get the thread pointer, |
735 |
// to ADD_TLS represents a call to .__get_tpointer to get the thread pointer, |
| 736 |
// later returning it into R3. |
736 |
// later returning it into R3. |
| 737 |
if (ADDTLSOp1Opcode == PPCISD::GET_TPOINTER) |
737 |
if (ADDTLSOp1Opcode == PPCISD::GET_TPOINTER) |
| 738 |
return true; |
738 |
return true; |
| 739 |
|
739 |
|
| 740 |
// The ADD_TLS note is explicitly acquiring the thread pointer (X13/R13). |
740 |
// The ADD_TLS note is explicitly acquiring the thread pointer (X13/R13). |
| 741 |
RegisterSDNode *AddFirstOpReg = |
741 |
RegisterSDNode *AddFirstOpReg = |
| 742 |
dyn_cast_or_null(ADDTLSOp1.getNode()); |
742 |
dyn_cast_or_null(ADDTLSOp1.getNode()); |
| 743 |
if (AddFirstOpReg && |
743 |
if (AddFirstOpReg && |
| 744 |
AddFirstOpReg->getReg() == Subtarget.getThreadPointerRegister()) |
744 |
AddFirstOpReg->getReg() == Subtarget.getThreadPointerRegister()) |
| 745 |
return true; |
745 |
return true; |
| 746 |
|
746 |
|
| 747 |
return false; |
747 |
return false; |
| 748 |
} |
748 |
} |
| 749 |
|
749 |
|
| 750 |
// canOptimizeTLSDFormToXForm - Optimize TLS accesses when an ADD_TLS |
750 |
// canOptimizeTLSDFormToXForm - Optimize TLS accesses when an ADD_TLS |
| 751 |
// instruction is present. An ADD_TLS instruction, followed by a D-Form memory |
751 |
// instruction is present. An ADD_TLS instruction, followed by a D-Form memory |
| 752 |
// operation, can be optimized to use an X-Form load or store, allowing the |
752 |
// operation, can be optimized to use an X-Form load or store, allowing the |
| 753 |
// ADD_TLS node to be removed completely. |
753 |
// ADD_TLS node to be removed completely. |
| 754 |
static bool canOptimizeTLSDFormToXForm(SelectionDAG *CurDAG, SDValue Base) { |
754 |
static bool canOptimizeTLSDFormToXForm(SelectionDAG *CurDAG, SDValue Base) { |
| 755 |
|
755 |
|
| 756 |
// Do not do this transformation at -O0. |
756 |
// Do not do this transformation at -O0. |
| 757 |
if (CurDAG->getTarget().getOptLevel() == CodeGenOpt::None) |
757 |
if (CurDAG->getTarget().getOptLevel() == CodeGenOpt::None) |
| 758 |
return false; |
758 |
return false; |
| 759 |
|
759 |
|
| 760 |
// In order to perform this optimization inside tryTLSXForm[Load|Store], |
760 |
// In order to perform this optimization inside tryTLSXForm[Load|Store], |
| 761 |
// Base is expected to be an ADD_TLS node. |
761 |
// Base is expected to be an ADD_TLS node. |
| 762 |
if (Base.getOpcode() != PPCISD::ADD_TLS) |
762 |
if (Base.getOpcode() != PPCISD::ADD_TLS) |
| 763 |
return false; |
763 |
return false; |
| 764 |
for (auto *ADDTLSUse : Base.getNode()->uses()) { |
764 |
for (auto *ADDTLSUse : Base.getNode()->uses()) { |
| 765 |
// The optimization to convert the D-Form load/store into its X-Form |
765 |
// The optimization to convert the D-Form load/store into its X-Form |
| 766 |
// counterpart should only occur if the source value offset of the load/ |
766 |
// counterpart should only occur if the source value offset of the load/ |
| 767 |
// store is 0. This also means that The offset should always be undefined. |
767 |
// store is 0. This also means that The offset should always be undefined. |
| 768 |
if (LoadSDNode *LD = dyn_cast(ADDTLSUse)) { |
768 |
if (LoadSDNode *LD = dyn_cast(ADDTLSUse)) { |
| 769 |
if (LD->getSrcValueOffset() != 0 || !LD->getOffset().isUndef()) |
769 |
if (LD->getSrcValueOffset() != 0 || !LD->getOffset().isUndef()) |
| 770 |
return false; |
770 |
return false; |
| 771 |
} else if (StoreSDNode *ST = dyn_cast(ADDTLSUse)) { |
771 |
} else if (StoreSDNode *ST = dyn_cast(ADDTLSUse)) { |
| 772 |
if (ST->getSrcValueOffset() != 0 || !ST->getOffset().isUndef()) |
772 |
if (ST->getSrcValueOffset() != 0 || !ST->getOffset().isUndef()) |
| 773 |
return false; |
773 |
return false; |
| 774 |
} else // Don't optimize if there are ADD_TLS users that aren't load/stores. |
774 |
} else // Don't optimize if there are ADD_TLS users that aren't load/stores. |
| 775 |
return false; |
775 |
return false; |
| 776 |
} |
776 |
} |
| 777 |
|
777 |
|
| 778 |
if (Base.getOperand(1).getOpcode() == PPCISD::TLS_LOCAL_EXEC_MAT_ADDR) |
778 |
if (Base.getOperand(1).getOpcode() == PPCISD::TLS_LOCAL_EXEC_MAT_ADDR) |
| 779 |
return false; |
779 |
return false; |
| 780 |
|
780 |
|
| 781 |
// Does the ADD_TLS node of the load/store use the thread pointer? |
781 |
// Does the ADD_TLS node of the load/store use the thread pointer? |
| 782 |
// If the thread pointer is not used as one of the operands of ADD_TLS, |
782 |
// If the thread pointer is not used as one of the operands of ADD_TLS, |
| 783 |
// then this optimization is not valid. |
783 |
// then this optimization is not valid. |
| 784 |
return isThreadPointerAcquisitionNode(Base, CurDAG); |
784 |
return isThreadPointerAcquisitionNode(Base, CurDAG); |
| 785 |
} |
785 |
} |
| 786 |
|
786 |
|
| 787 |
bool PPCDAGToDAGISel::tryTLSXFormStore(StoreSDNode *ST) { |
787 |
bool PPCDAGToDAGISel::tryTLSXFormStore(StoreSDNode *ST) { |
| 788 |
SDValue Base = ST->getBasePtr(); |
788 |
SDValue Base = ST->getBasePtr(); |
| 789 |
if (!canOptimizeTLSDFormToXForm(CurDAG, Base)) |
789 |
if (!canOptimizeTLSDFormToXForm(CurDAG, Base)) |
| 790 |
return false; |
790 |
return false; |
| 791 |
|
791 |
|
| 792 |
SDLoc dl(ST); |
792 |
SDLoc dl(ST); |
| 793 |
EVT MemVT = ST->getMemoryVT(); |
793 |
EVT MemVT = ST->getMemoryVT(); |
| 794 |
EVT RegVT = ST->getValue().getValueType(); |
794 |
EVT RegVT = ST->getValue().getValueType(); |
| 795 |
|
795 |
|
| 796 |
unsigned Opcode; |
796 |
unsigned Opcode; |
| 797 |
switch (MemVT.getSimpleVT().SimpleTy) { |
797 |
switch (MemVT.getSimpleVT().SimpleTy) { |
| 798 |
default: |
798 |
default: |
| 799 |
return false; |
799 |
return false; |
| 800 |
case MVT::i8: { |
800 |
case MVT::i8: { |
| 801 |
Opcode = (RegVT == MVT::i32) ? PPC::STBXTLS_32 : PPC::STBXTLS; |
801 |
Opcode = (RegVT == MVT::i32) ? PPC::STBXTLS_32 : PPC::STBXTLS; |
| 802 |
break; |
802 |
break; |
| 803 |
} |
803 |
} |
| 804 |
case MVT::i16: { |
804 |
case MVT::i16: { |
| 805 |
Opcode = (RegVT == MVT::i32) ? PPC::STHXTLS_32 : PPC::STHXTLS; |
805 |
Opcode = (RegVT == MVT::i32) ? PPC::STHXTLS_32 : PPC::STHXTLS; |
| 806 |
break; |
806 |
break; |
| 807 |
} |
807 |
} |
| 808 |
case MVT::i32: { |
808 |
case MVT::i32: { |
| 809 |
Opcode = (RegVT == MVT::i32) ? PPC::STWXTLS_32 : PPC::STWXTLS; |
809 |
Opcode = (RegVT == MVT::i32) ? PPC::STWXTLS_32 : PPC::STWXTLS; |
| 810 |
break; |
810 |
break; |
| 811 |
} |
811 |
} |
| 812 |
case MVT::i64: { |
812 |
case MVT::i64: { |
| 813 |
Opcode = PPC::STDXTLS; |
813 |
Opcode = PPC::STDXTLS; |
| 814 |
break; |
814 |
break; |
| 815 |
} |
815 |
} |
| 816 |
case MVT::f32: { |
816 |
case MVT::f32: { |
| 817 |
Opcode = PPC::STFSXTLS; |
817 |
Opcode = PPC::STFSXTLS; |
| 818 |
break; |
818 |
break; |
| 819 |
} |
819 |
} |
| 820 |
case MVT::f64: { |
820 |
case MVT::f64: { |
| 821 |
Opcode = PPC::STFDXTLS; |
821 |
Opcode = PPC::STFDXTLS; |
| 822 |
break; |
822 |
break; |
| 823 |
} |
823 |
} |
| 824 |
} |
824 |
} |
| 825 |
SDValue Chain = ST->getChain(); |
825 |
SDValue Chain = ST->getChain(); |
| 826 |
SDVTList VTs = ST->getVTList(); |
826 |
SDVTList VTs = ST->getVTList(); |
| 827 |
SDValue Ops[] = {ST->getValue(), Base.getOperand(0), Base.getOperand(1), |
827 |
SDValue Ops[] = {ST->getValue(), Base.getOperand(0), Base.getOperand(1), |
| 828 |
Chain}; |
828 |
Chain}; |
| 829 |
SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops); |
829 |
SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops); |
| 830 |
transferMemOperands(ST, MN); |
830 |
transferMemOperands(ST, MN); |
| 831 |
ReplaceNode(ST, MN); |
831 |
ReplaceNode(ST, MN); |
| 832 |
return true; |
832 |
return true; |
| 833 |
} |
833 |
} |
| 834 |
|
834 |
|
| 835 |
bool PPCDAGToDAGISel::tryTLSXFormLoad(LoadSDNode *LD) { |
835 |
bool PPCDAGToDAGISel::tryTLSXFormLoad(LoadSDNode *LD) { |
| 836 |
SDValue Base = LD->getBasePtr(); |
836 |
SDValue Base = LD->getBasePtr(); |
| 837 |
if (!canOptimizeTLSDFormToXForm(CurDAG, Base)) |
837 |
if (!canOptimizeTLSDFormToXForm(CurDAG, Base)) |
| 838 |
return false; |
838 |
return false; |
| 839 |
|
839 |
|
| 840 |
SDLoc dl(LD); |
840 |
SDLoc dl(LD); |
| 841 |
EVT MemVT = LD->getMemoryVT(); |
841 |
EVT MemVT = LD->getMemoryVT(); |
| 842 |
EVT RegVT = LD->getValueType(0); |
842 |
EVT RegVT = LD->getValueType(0); |
| 843 |
bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; |
843 |
bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; |
| 844 |
unsigned Opcode; |
844 |
unsigned Opcode; |
| 845 |
switch (MemVT.getSimpleVT().SimpleTy) { |
845 |
switch (MemVT.getSimpleVT().SimpleTy) { |
| 846 |
default: |
846 |
default: |
| 847 |
return false; |
847 |
return false; |
| 848 |
case MVT::i8: { |
848 |
case MVT::i8: { |
| 849 |
Opcode = (RegVT == MVT::i32) ? PPC::LBZXTLS_32 : PPC::LBZXTLS; |
849 |
Opcode = (RegVT == MVT::i32) ? PPC::LBZXTLS_32 : PPC::LBZXTLS; |
| 850 |
break; |
850 |
break; |
| 851 |
} |
851 |
} |
| 852 |
case MVT::i16: { |
852 |
case MVT::i16: { |
| 853 |
if (RegVT == MVT::i32) |
853 |
if (RegVT == MVT::i32) |
| 854 |
Opcode = isSExt ? PPC::LHAXTLS_32 : PPC::LHZXTLS_32; |
854 |
Opcode = isSExt ? PPC::LHAXTLS_32 : PPC::LHZXTLS_32; |
| 855 |
else |
855 |
else |
| 856 |
Opcode = isSExt ? PPC::LHAXTLS : PPC::LHZXTLS; |
856 |
Opcode = isSExt ? PPC::LHAXTLS : PPC::LHZXTLS; |
| 857 |
break; |
857 |
break; |
| 858 |
} |
858 |
} |
| 859 |
case MVT::i32: { |
859 |
case MVT::i32: { |
| 860 |
if (RegVT == MVT::i32) |
860 |
if (RegVT == MVT::i32) |
| 861 |
Opcode = isSExt ? PPC::LWAXTLS_32 : PPC::LWZXTLS_32; |
861 |
Opcode = isSExt ? PPC::LWAXTLS_32 : PPC::LWZXTLS_32; |
| 862 |
else |
862 |
else |
| 863 |
Opcode = isSExt ? PPC::LWAXTLS : PPC::LWZXTLS; |
863 |
Opcode = isSExt ? PPC::LWAXTLS : PPC::LWZXTLS; |
| 864 |
break; |
864 |
break; |
| 865 |
} |
865 |
} |
| 866 |
case MVT::i64: { |
866 |
case MVT::i64: { |
| 867 |
Opcode = PPC::LDXTLS; |
867 |
Opcode = PPC::LDXTLS; |
| 868 |
break; |
868 |
break; |
| 869 |
} |
869 |
} |
| 870 |
case MVT::f32: { |
870 |
case MVT::f32: { |
| 871 |
Opcode = PPC::LFSXTLS; |
871 |
Opcode = PPC::LFSXTLS; |
| 872 |
break; |
872 |
break; |
| 873 |
} |
873 |
} |
| 874 |
case MVT::f64: { |
874 |
case MVT::f64: { |
| 875 |
Opcode = PPC::LFDXTLS; |
875 |
Opcode = PPC::LFDXTLS; |
| 876 |
break; |
876 |
break; |
| 877 |
} |
877 |
} |
| 878 |
} |
878 |
} |
| 879 |
SDValue Chain = LD->getChain(); |
879 |
SDValue Chain = LD->getChain(); |
| 880 |
SDVTList VTs = LD->getVTList(); |
880 |
SDVTList VTs = LD->getVTList(); |
| 881 |
SDValue Ops[] = {Base.getOperand(0), Base.getOperand(1), Chain}; |
881 |
SDValue Ops[] = {Base.getOperand(0), Base.getOperand(1), Chain}; |
| 882 |
SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops); |
882 |
SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops); |
| 883 |
transferMemOperands(LD, MN); |
883 |
transferMemOperands(LD, MN); |
| 884 |
ReplaceNode(LD, MN); |
884 |
ReplaceNode(LD, MN); |
| 885 |
return true; |
885 |
return true; |
| 886 |
} |
886 |
} |
| 887 |
|
887 |
|
| 888 |
/// Turn an or of two masked values into the rotate left word immediate then |
888 |
/// Turn an or of two masked values into the rotate left word immediate then |
| 889 |
/// mask insert (rlwimi) instruction. |
889 |
/// mask insert (rlwimi) instruction. |
| 890 |
bool PPCDAGToDAGISel::tryBitfieldInsert(SDNode *N) { |
890 |
bool PPCDAGToDAGISel::tryBitfieldInsert(SDNode *N) { |
| 891 |
SDValue Op0 = N->getOperand(0); |
891 |
SDValue Op0 = N->getOperand(0); |
| 892 |
SDValue Op1 = N->getOperand(1); |
892 |
SDValue Op1 = N->getOperand(1); |
| 893 |
SDLoc dl(N); |
893 |
SDLoc dl(N); |
| 894 |
|
894 |
|
| 895 |
KnownBits LKnown = CurDAG->computeKnownBits(Op0); |
895 |
KnownBits LKnown = CurDAG->computeKnownBits(Op0); |
| 896 |
KnownBits RKnown = CurDAG->computeKnownBits(Op1); |
896 |
KnownBits RKnown = CurDAG->computeKnownBits(Op1); |
| 897 |
|
897 |
|
| 898 |
unsigned TargetMask = LKnown.Zero.getZExtValue(); |
898 |
unsigned TargetMask = LKnown.Zero.getZExtValue(); |
| 899 |
unsigned InsertMask = RKnown.Zero.getZExtValue(); |
899 |
unsigned InsertMask = RKnown.Zero.getZExtValue(); |
| 900 |
|
900 |
|
| 901 |
if ((TargetMask | InsertMask) == 0xFFFFFFFF) { |
901 |
if ((TargetMask | InsertMask) == 0xFFFFFFFF) { |
| 902 |
unsigned Op0Opc = Op0.getOpcode(); |
902 |
unsigned Op0Opc = Op0.getOpcode(); |
| 903 |
unsigned Op1Opc = Op1.getOpcode(); |
903 |
unsigned Op1Opc = Op1.getOpcode(); |
| 904 |
unsigned Value, SH = 0; |
904 |
unsigned Value, SH = 0; |
| 905 |
TargetMask = ~TargetMask; |
905 |
TargetMask = ~TargetMask; |
| 906 |
InsertMask = ~InsertMask; |
906 |
InsertMask = ~InsertMask; |
| 907 |
|
907 |
|
| 908 |
// If the LHS has a foldable shift and the RHS does not, then swap it to the |
908 |
// If the LHS has a foldable shift and the RHS does not, then swap it to the |
| 909 |
// RHS so that we can fold the shift into the insert. |
909 |
// RHS so that we can fold the shift into the insert. |
| 910 |
if (Op0Opc == ISD::AND && Op1Opc == ISD::AND) { |
910 |
if (Op0Opc == ISD::AND && Op1Opc == ISD::AND) { |
| 911 |
if (Op0.getOperand(0).getOpcode() == ISD::SHL || |
911 |
if (Op0.getOperand(0).getOpcode() == ISD::SHL || |
| 912 |
Op0.getOperand(0).getOpcode() == ISD::SRL) { |
912 |
Op0.getOperand(0).getOpcode() == ISD::SRL) { |
| 913 |
if (Op1.getOperand(0).getOpcode() != ISD::SHL && |
913 |
if (Op1.getOperand(0).getOpcode() != ISD::SHL && |
| 914 |
Op1.getOperand(0).getOpcode() != ISD::SRL) { |
914 |
Op1.getOperand(0).getOpcode() != ISD::SRL) { |
| 915 |
std::swap(Op0, Op1); |
915 |
std::swap(Op0, Op1); |
| 916 |
std::swap(Op0Opc, Op1Opc); |
916 |
std::swap(Op0Opc, Op1Opc); |
| 917 |
std::swap(TargetMask, InsertMask); |
917 |
std::swap(TargetMask, InsertMask); |
| 918 |
} |
918 |
} |
| 919 |
} |
919 |
} |
| 920 |
} else if (Op0Opc == ISD::SHL || Op0Opc == ISD::SRL) { |
920 |
} else if (Op0Opc == ISD::SHL || Op0Opc == ISD::SRL) { |
| 921 |
if (Op1Opc == ISD::AND && Op1.getOperand(0).getOpcode() != ISD::SHL && |
921 |
if (Op1Opc == ISD::AND && Op1.getOperand(0).getOpcode() != ISD::SHL && |
| 922 |
Op1.getOperand(0).getOpcode() != ISD::SRL) { |
922 |
Op1.getOperand(0).getOpcode() != ISD::SRL) { |
| 923 |
std::swap(Op0, Op1); |
923 |
std::swap(Op0, Op1); |
| 924 |
std::swap(Op0Opc, Op1Opc); |
924 |
std::swap(Op0Opc, Op1Opc); |
| 925 |
std::swap(TargetMask, InsertMask); |
925 |
std::swap(TargetMask, InsertMask); |
| 926 |
} |
926 |
} |
| 927 |
} |
927 |
} |
| 928 |
|
928 |
|
| 929 |
unsigned MB, ME; |
929 |
unsigned MB, ME; |
| 930 |
if (isRunOfOnes(InsertMask, MB, ME)) { |
930 |
if (isRunOfOnes(InsertMask, MB, ME)) { |
| 931 |
if ((Op1Opc == ISD::SHL || Op1Opc == ISD::SRL) && |
931 |
if ((Op1Opc == ISD::SHL || Op1Opc == ISD::SRL) && |
| 932 |
isInt32Immediate(Op1.getOperand(1), Value)) { |
932 |
isInt32Immediate(Op1.getOperand(1), Value)) { |
| 933 |
Op1 = Op1.getOperand(0); |
933 |
Op1 = Op1.getOperand(0); |
| 934 |
SH = (Op1Opc == ISD::SHL) ? Value : 32 - Value; |
934 |
SH = (Op1Opc == ISD::SHL) ? Value : 32 - Value; |
| 935 |
} |
935 |
} |
| 936 |
if (Op1Opc == ISD::AND) { |
936 |
if (Op1Opc == ISD::AND) { |
| 937 |
// The AND mask might not be a constant, and we need to make sure that |
937 |
// The AND mask might not be a constant, and we need to make sure that |
| 938 |
// if we're going to fold the masking with the insert, all bits not |
938 |
// if we're going to fold the masking with the insert, all bits not |
| 939 |
// know to be zero in the mask are known to be one. |
939 |
// know to be zero in the mask are known to be one. |
| 940 |
KnownBits MKnown = CurDAG->computeKnownBits(Op1.getOperand(1)); |
940 |
KnownBits MKnown = CurDAG->computeKnownBits(Op1.getOperand(1)); |
| 941 |
bool CanFoldMask = InsertMask == MKnown.One.getZExtValue(); |
941 |
bool CanFoldMask = InsertMask == MKnown.One.getZExtValue(); |
| 942 |
|
942 |
|
| 943 |
unsigned SHOpc = Op1.getOperand(0).getOpcode(); |
943 |
unsigned SHOpc = Op1.getOperand(0).getOpcode(); |
| 944 |
if ((SHOpc == ISD::SHL || SHOpc == ISD::SRL) && CanFoldMask && |
944 |
if ((SHOpc == ISD::SHL || SHOpc == ISD::SRL) && CanFoldMask && |
| 945 |
isInt32Immediate(Op1.getOperand(0).getOperand(1), Value)) { |
945 |
isInt32Immediate(Op1.getOperand(0).getOperand(1), Value)) { |
| 946 |
// Note that Value must be in range here (less than 32) because |
946 |
// Note that Value must be in range here (less than 32) because |
| 947 |
// otherwise there would not be any bits set in InsertMask. |
947 |
// otherwise there would not be any bits set in InsertMask. |
| 948 |
Op1 = Op1.getOperand(0).getOperand(0); |
948 |
Op1 = Op1.getOperand(0).getOperand(0); |
| 949 |
SH = (SHOpc == ISD::SHL) ? Value : 32 - Value; |
949 |
SH = (SHOpc == ISD::SHL) ? Value : 32 - Value; |
| 950 |
} |
950 |
} |
| 951 |
} |
951 |
} |
| 952 |
|
952 |
|
| 953 |
SH &= 31; |
953 |
SH &= 31; |
| 954 |
SDValue Ops[] = { Op0, Op1, getI32Imm(SH, dl), getI32Imm(MB, dl), |
954 |
SDValue Ops[] = { Op0, Op1, getI32Imm(SH, dl), getI32Imm(MB, dl), |
| 955 |
getI32Imm(ME, dl) }; |
955 |
getI32Imm(ME, dl) }; |
| 956 |
ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops)); |
956 |
ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops)); |
| 957 |
return true; |
957 |
return true; |
| 958 |
} |
958 |
} |
| 959 |
} |
959 |
} |
| 960 |
return false; |
960 |
return false; |
| 961 |
} |
961 |
} |
| 962 |
|
962 |
|
| 963 |
static unsigned allUsesTruncate(SelectionDAG *CurDAG, SDNode *N) { |
963 |
static unsigned allUsesTruncate(SelectionDAG *CurDAG, SDNode *N) { |
| 964 |
unsigned MaxTruncation = 0; |
964 |
unsigned MaxTruncation = 0; |
| 965 |
// Cannot use range-based for loop here as we need the actual use (i.e. we |
965 |
// Cannot use range-based for loop here as we need the actual use (i.e. we |
| 966 |
// need the operand number corresponding to the use). A range-based for |
966 |
// need the operand number corresponding to the use). A range-based for |
| 967 |
// will unbox the use and provide an SDNode*. |
967 |
// will unbox the use and provide an SDNode*. |
| 968 |
for (SDNode::use_iterator Use = N->use_begin(), UseEnd = N->use_end(); |
968 |
for (SDNode::use_iterator Use = N->use_begin(), UseEnd = N->use_end(); |
| 969 |
Use != UseEnd; ++Use) { |
969 |
Use != UseEnd; ++Use) { |
| 970 |
unsigned Opc = |
970 |
unsigned Opc = |
| 971 |
Use->isMachineOpcode() ? Use->getMachineOpcode() : Use->getOpcode(); |
971 |
Use->isMachineOpcode() ? Use->getMachineOpcode() : Use->getOpcode(); |
| 972 |
switch (Opc) { |
972 |
switch (Opc) { |
| 973 |
default: return 0; |
973 |
default: return 0; |
| 974 |
case ISD::TRUNCATE: |
974 |
case ISD::TRUNCATE: |
| 975 |
if (Use->isMachineOpcode()) |
975 |
if (Use->isMachineOpcode()) |
| 976 |
return 0; |
976 |
return 0; |
| 977 |
MaxTruncation = |
977 |
MaxTruncation = |
| 978 |
std::max(MaxTruncation, (unsigned)Use->getValueType(0).getSizeInBits()); |
978 |
std::max(MaxTruncation, (unsigned)Use->getValueType(0).getSizeInBits()); |
| 979 |
continue; |
979 |
continue; |
| 980 |
case ISD::STORE: { |
980 |
case ISD::STORE: { |
| 981 |
if (Use->isMachineOpcode()) |
981 |
if (Use->isMachineOpcode()) |
| 982 |
return 0; |
982 |
return 0; |
| 983 |
StoreSDNode *STN = cast(*Use); |
983 |
StoreSDNode *STN = cast(*Use); |
| 984 |
unsigned MemVTSize = STN->getMemoryVT().getSizeInBits(); |
984 |
unsigned MemVTSize = STN->getMemoryVT().getSizeInBits(); |
| 985 |
if (MemVTSize == 64 || Use.getOperandNo() != 0) |
985 |
if (MemVTSize == 64 || Use.getOperandNo() != 0) |
| 986 |
return 0; |
986 |
return 0; |
| 987 |
MaxTruncation = std::max(MaxTruncation, MemVTSize); |
987 |
MaxTruncation = std::max(MaxTruncation, MemVTSize); |
| 988 |
continue; |
988 |
continue; |
| 989 |
} |
989 |
} |
| 990 |
case PPC::STW8: |
990 |
case PPC::STW8: |
| 991 |
case PPC::STWX8: |
991 |
case PPC::STWX8: |
| 992 |
case PPC::STWU8: |
992 |
case PPC::STWU8: |
| 993 |
case PPC::STWUX8: |
993 |
case PPC::STWUX8: |
| 994 |
if (Use.getOperandNo() != 0) |
994 |
if (Use.getOperandNo() != 0) |
| 995 |
return 0; |
995 |
return 0; |
| 996 |
MaxTruncation = std::max(MaxTruncation, 32u); |
996 |
MaxTruncation = std::max(MaxTruncation, 32u); |
| 997 |
continue; |
997 |
continue; |
| 998 |
case PPC::STH8: |
998 |
case PPC::STH8: |
| 999 |
case PPC::STHX8: |
999 |
case PPC::STHX8: |
| 1000 |
case PPC::STHU8: |
1000 |
case PPC::STHU8: |
| 1001 |
case PPC::STHUX8: |
1001 |
case PPC::STHUX8: |
| 1002 |
if (Use.getOperandNo() != 0) |
1002 |
if (Use.getOperandNo() != 0) |
| 1003 |
return 0; |
1003 |
return 0; |
| 1004 |
MaxTruncation = std::max(MaxTruncation, 16u); |
1004 |
MaxTruncation = std::max(MaxTruncation, 16u); |
| 1005 |
continue; |
1005 |
continue; |
| 1006 |
case PPC::STB8: |
1006 |
case PPC::STB8: |
| 1007 |
case PPC::STBX8: |
1007 |
case PPC::STBX8: |
| 1008 |
case PPC::STBU8: |
1008 |
case PPC::STBU8: |
| 1009 |
case PPC::STBUX8: |
1009 |
case PPC::STBUX8: |
| 1010 |
if (Use.getOperandNo() != 0) |
1010 |
if (Use.getOperandNo() != 0) |
| 1011 |
return 0; |
1011 |
return 0; |
| 1012 |
MaxTruncation = std::max(MaxTruncation, 8u); |
1012 |
MaxTruncation = std::max(MaxTruncation, 8u); |
| 1013 |
continue; |
1013 |
continue; |
| 1014 |
} |
1014 |
} |
| 1015 |
} |
1015 |
} |
| 1016 |
return MaxTruncation; |
1016 |
return MaxTruncation; |
| 1017 |
} |
1017 |
} |
| 1018 |
|
1018 |
|
| 1019 |
// For any 32 < Num < 64, check if the Imm contains at least Num consecutive |
1019 |
// For any 32 < Num < 64, check if the Imm contains at least Num consecutive |
| 1020 |
// zeros and return the number of bits by the left of these consecutive zeros. |
1020 |
// zeros and return the number of bits by the left of these consecutive zeros. |
| 1021 |
static int findContiguousZerosAtLeast(uint64_t Imm, unsigned Num) { |
1021 |
static int findContiguousZerosAtLeast(uint64_t Imm, unsigned Num) { |
| 1022 |
unsigned HiTZ = llvm::countr_zero(Hi_32(Imm)); |
1022 |
unsigned HiTZ = llvm::countr_zero(Hi_32(Imm)); |
| 1023 |
unsigned LoLZ = llvm::countl_zero(Lo_32(Imm)); |
1023 |
unsigned LoLZ = llvm::countl_zero(Lo_32(Imm)); |
| 1024 |
if ((HiTZ + LoLZ) >= Num) |
1024 |
if ((HiTZ + LoLZ) >= Num) |
| 1025 |
return (32 + HiTZ); |
1025 |
return (32 + HiTZ); |
| 1026 |
return 0; |
1026 |
return 0; |
| 1027 |
} |
1027 |
} |
| 1028 |
|
1028 |
|
| 1029 |
// Direct materialization of 64-bit constants by enumerated patterns. |
1029 |
// Direct materialization of 64-bit constants by enumerated patterns. |
| 1030 |
static SDNode *selectI64ImmDirect(SelectionDAG *CurDAG, const SDLoc &dl, |
1030 |
static SDNode *selectI64ImmDirect(SelectionDAG *CurDAG, const SDLoc &dl, |
| 1031 |
uint64_t Imm, unsigned &InstCnt) { |
1031 |
uint64_t Imm, unsigned &InstCnt) { |
| 1032 |
unsigned TZ = llvm::countr_zero(Imm); |
1032 |
unsigned TZ = llvm::countr_zero(Imm); |
| 1033 |
unsigned LZ = llvm::countl_zero(Imm); |
1033 |
unsigned LZ = llvm::countl_zero(Imm); |
| 1034 |
unsigned TO = llvm::countr_one(Imm); |
1034 |
unsigned TO = llvm::countr_one(Imm); |
| 1035 |
unsigned LO = llvm::countl_one(Imm); |
1035 |
unsigned LO = llvm::countl_one(Imm); |
| 1036 |
unsigned Hi32 = Hi_32(Imm); |
1036 |
unsigned Hi32 = Hi_32(Imm); |
| 1037 |
unsigned Lo32 = Lo_32(Imm); |
1037 |
unsigned Lo32 = Lo_32(Imm); |
| 1038 |
SDNode *Result = nullptr; |
1038 |
SDNode *Result = nullptr; |
| 1039 |
unsigned Shift = 0; |
1039 |
unsigned Shift = 0; |
| 1040 |
|
1040 |
|
| 1041 |
auto getI32Imm = [CurDAG, dl](unsigned Imm) { |
1041 |
auto getI32Imm = [CurDAG, dl](unsigned Imm) { |
| 1042 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
1042 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
| 1043 |
}; |
1043 |
}; |
| 1044 |
|
1044 |
|
| 1045 |
// Following patterns use 1 instructions to materialize the Imm. |
1045 |
// Following patterns use 1 instructions to materialize the Imm. |
| 1046 |
InstCnt = 1; |
1046 |
InstCnt = 1; |
| 1047 |
// 1-1) Patterns : {zeros}{15-bit valve} |
1047 |
// 1-1) Patterns : {zeros}{15-bit valve} |
| 1048 |
// {ones}{15-bit valve} |
1048 |
// {ones}{15-bit valve} |
| 1049 |
if (isInt<16>(Imm)) { |
1049 |
if (isInt<16>(Imm)) { |
| 1050 |
SDValue SDImm = CurDAG->getTargetConstant(Imm, dl, MVT::i64); |
1050 |
SDValue SDImm = CurDAG->getTargetConstant(Imm, dl, MVT::i64); |
| 1051 |
return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm); |
1051 |
return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm); |
| 1052 |
} |
1052 |
} |
| 1053 |
// 1-2) Patterns : {zeros}{15-bit valve}{16 zeros} |
1053 |
// 1-2) Patterns : {zeros}{15-bit valve}{16 zeros} |
| 1054 |
// {ones}{15-bit valve}{16 zeros} |
1054 |
// {ones}{15-bit valve}{16 zeros} |
| 1055 |
if (TZ > 15 && (LZ > 32 || LO > 32)) |
1055 |
if (TZ > 15 && (LZ > 32 || LO > 32)) |
| 1056 |
return CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, |
1056 |
return CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, |
| 1057 |
getI32Imm((Imm >> 16) & 0xffff)); |
1057 |
getI32Imm((Imm >> 16) & 0xffff)); |
| 1058 |
|
1058 |
|
| 1059 |
// Following patterns use 2 instructions to materialize the Imm. |
1059 |
// Following patterns use 2 instructions to materialize the Imm. |
| 1060 |
InstCnt = 2; |
1060 |
InstCnt = 2; |
| 1061 |
assert(LZ < 64 && "Unexpected leading zeros here."); |
1061 |
assert(LZ < 64 && "Unexpected leading zeros here."); |
| 1062 |
// Count of ones follwing the leading zeros. |
1062 |
// Count of ones follwing the leading zeros. |
| 1063 |
unsigned FO = llvm::countl_one(Imm << LZ); |
1063 |
unsigned FO = llvm::countl_one(Imm << LZ); |
| 1064 |
// 2-1) Patterns : {zeros}{31-bit value} |
1064 |
// 2-1) Patterns : {zeros}{31-bit value} |
| 1065 |
// {ones}{31-bit value} |
1065 |
// {ones}{31-bit value} |
| 1066 |
if (isInt<32>(Imm)) { |
1066 |
if (isInt<32>(Imm)) { |
| 1067 |
uint64_t ImmHi16 = (Imm >> 16) & 0xffff; |
1067 |
uint64_t ImmHi16 = (Imm >> 16) & 0xffff; |
| 1068 |
unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; |
1068 |
unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; |
| 1069 |
Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); |
1069 |
Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); |
| 1070 |
return CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
1070 |
return CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
| 1071 |
getI32Imm(Imm & 0xffff)); |
1071 |
getI32Imm(Imm & 0xffff)); |
| 1072 |
} |
1072 |
} |
| 1073 |
// 2-2) Patterns : {zeros}{ones}{15-bit value}{zeros} |
1073 |
// 2-2) Patterns : {zeros}{ones}{15-bit value}{zeros} |
| 1074 |
// {zeros}{15-bit value}{zeros} |
1074 |
// {zeros}{15-bit value}{zeros} |
| 1075 |
// {zeros}{ones}{15-bit value} |
1075 |
// {zeros}{ones}{15-bit value} |
| 1076 |
// {ones}{15-bit value}{zeros} |
1076 |
// {ones}{15-bit value}{zeros} |
| 1077 |
// We can take advantage of LI's sign-extension semantics to generate leading |
1077 |
// We can take advantage of LI's sign-extension semantics to generate leading |
| 1078 |
// ones, and then use RLDIC to mask off the ones in both sides after rotation. |
1078 |
// ones, and then use RLDIC to mask off the ones in both sides after rotation. |
| 1079 |
if ((LZ + FO + TZ) > 48) { |
1079 |
if ((LZ + FO + TZ) > 48) { |
| 1080 |
Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
1080 |
Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
| 1081 |
getI32Imm((Imm >> TZ) & 0xffff)); |
1081 |
getI32Imm((Imm >> TZ) & 0xffff)); |
| 1082 |
return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), |
1082 |
return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), |
| 1083 |
getI32Imm(TZ), getI32Imm(LZ)); |
1083 |
getI32Imm(TZ), getI32Imm(LZ)); |
| 1084 |
} |
1084 |
} |
| 1085 |
// 2-3) Pattern : {zeros}{15-bit value}{ones} |
1085 |
// 2-3) Pattern : {zeros}{15-bit value}{ones} |
| 1086 |
// Shift right the Imm by (48 - LZ) bits to construct a negtive 16 bits value, |
1086 |
// Shift right the Imm by (48 - LZ) bits to construct a negtive 16 bits value, |
| 1087 |
// therefore we can take advantage of LI's sign-extension semantics, and then |
1087 |
// therefore we can take advantage of LI's sign-extension semantics, and then |
| 1088 |
// mask them off after rotation. |
1088 |
// mask them off after rotation. |
| 1089 |
// |
1089 |
// |
| 1090 |
// +--LZ--||-15-bit-||--TO--+ +-------------|--16-bit--+ |
1090 |
// +--LZ--||-15-bit-||--TO--+ +-------------|--16-bit--+ |
| 1091 |
// |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1| |
1091 |
// |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1| |
| 1092 |
// +------------------------+ +------------------------+ |
1092 |
// +------------------------+ +------------------------+ |
| 1093 |
// 63 0 63 0 |
1093 |
// 63 0 63 0 |
| 1094 |
// Imm (Imm >> (48 - LZ) & 0xffff) |
1094 |
// Imm (Imm >> (48 - LZ) & 0xffff) |
| 1095 |
// +----sext-----|--16-bit--+ +clear-|-----------------+ |
1095 |
// +----sext-----|--16-bit--+ +clear-|-----------------+ |
| 1096 |
// |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111| |
1096 |
// |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111| |
| 1097 |
// +------------------------+ +------------------------+ |
1097 |
// +------------------------+ +------------------------+ |
| 1098 |
// 63 0 63 0 |
1098 |
// 63 0 63 0 |
| 1099 |
// LI8: sext many leading zeros RLDICL: rotate left (48 - LZ), clear left LZ |
1099 |
// LI8: sext many leading zeros RLDICL: rotate left (48 - LZ), clear left LZ |
| 1100 |
if ((LZ + TO) > 48) { |
1100 |
if ((LZ + TO) > 48) { |
| 1101 |
// Since the immediates with (LZ > 32) have been handled by previous |
1101 |
// Since the immediates with (LZ > 32) have been handled by previous |
| 1102 |
// patterns, here we have (LZ <= 32) to make sure we will not shift right |
1102 |
// patterns, here we have (LZ <= 32) to make sure we will not shift right |
| 1103 |
// the Imm by a negative value. |
1103 |
// the Imm by a negative value. |
| 1104 |
assert(LZ <= 32 && "Unexpected shift value."); |
1104 |
assert(LZ <= 32 && "Unexpected shift value."); |
| 1105 |
Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
1105 |
Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
| 1106 |
getI32Imm((Imm >> (48 - LZ) & 0xffff))); |
1106 |
getI32Imm((Imm >> (48 - LZ) & 0xffff))); |
| 1107 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1107 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
| 1108 |
getI32Imm(48 - LZ), getI32Imm(LZ)); |
1108 |
getI32Imm(48 - LZ), getI32Imm(LZ)); |
| 1109 |
} |
1109 |
} |
| 1110 |
// 2-4) Patterns : {zeros}{ones}{15-bit value}{ones} |
1110 |
// 2-4) Patterns : {zeros}{ones}{15-bit value}{ones} |
| 1111 |
// {ones}{15-bit value}{ones} |
1111 |
// {ones}{15-bit value}{ones} |
| 1112 |
// We can take advantage of LI's sign-extension semantics to generate leading |
1112 |
// We can take advantage of LI's sign-extension semantics to generate leading |
| 1113 |
// ones, and then use RLDICL to mask off the ones in left sides (if required) |
1113 |
// ones, and then use RLDICL to mask off the ones in left sides (if required) |
| 1114 |
// after rotation. |
1114 |
// after rotation. |
| 1115 |
// |
1115 |
// |
| 1116 |
// +-LZ-FO||-15-bit-||--TO--+ +-------------|--16-bit--+ |
1116 |
// +-LZ-FO||-15-bit-||--TO--+ +-------------|--16-bit--+ |
| 1117 |
// |00011110bbbbbbbbb1111111| -> |000000000011110bbbbbbbbb| |
1117 |
// |00011110bbbbbbbbb1111111| -> |000000000011110bbbbbbbbb| |
| 1118 |
// +------------------------+ +------------------------+ |
1118 |
// +------------------------+ +------------------------+ |
| 1119 |
// 63 0 63 0 |
1119 |
// 63 0 63 0 |
| 1120 |
// Imm (Imm >> TO) & 0xffff |
1120 |
// Imm (Imm >> TO) & 0xffff |
| 1121 |
// +----sext-----|--16-bit--+ +LZ|---------------------+ |
1121 |
// +----sext-----|--16-bit--+ +LZ|---------------------+ |
| 1122 |
// |111111111111110bbbbbbbbb| -> |00011110bbbbbbbbb1111111| |
1122 |
// |111111111111110bbbbbbbbb| -> |00011110bbbbbbbbb1111111| |
| 1123 |
// +------------------------+ +------------------------+ |
1123 |
// +------------------------+ +------------------------+ |
| 1124 |
// 63 0 63 0 |
1124 |
// 63 0 63 0 |
| 1125 |
// LI8: sext many leading zeros RLDICL: rotate left TO, clear left LZ |
1125 |
// LI8: sext many leading zeros RLDICL: rotate left TO, clear left LZ |
| 1126 |
if ((LZ + FO + TO) > 48) { |
1126 |
if ((LZ + FO + TO) > 48) { |
| 1127 |
Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
1127 |
Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
| 1128 |
getI32Imm((Imm >> TO) & 0xffff)); |
1128 |
getI32Imm((Imm >> TO) & 0xffff)); |
| 1129 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1129 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
| 1130 |
getI32Imm(TO), getI32Imm(LZ)); |
1130 |
getI32Imm(TO), getI32Imm(LZ)); |
| 1131 |
} |
1131 |
} |
| 1132 |
// 2-5) Pattern : {32 zeros}{****}{0}{15-bit value} |
1132 |
// 2-5) Pattern : {32 zeros}{****}{0}{15-bit value} |
| 1133 |
// If Hi32 is zero and the Lo16(in Lo32) can be presented as a positive 16 bit |
1133 |
// If Hi32 is zero and the Lo16(in Lo32) can be presented as a positive 16 bit |
| 1134 |
// value, we can use LI for Lo16 without generating leading ones then add the |
1134 |
// value, we can use LI for Lo16 without generating leading ones then add the |
| 1135 |
// Hi16(in Lo32). |
1135 |
// Hi16(in Lo32). |
| 1136 |
if (LZ == 32 && ((Lo32 & 0x8000) == 0)) { |
1136 |
if (LZ == 32 && ((Lo32 & 0x8000) == 0)) { |
| 1137 |
Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
1137 |
Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
| 1138 |
getI32Imm(Lo32 & 0xffff)); |
1138 |
getI32Imm(Lo32 & 0xffff)); |
| 1139 |
return CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, SDValue(Result, 0), |
1139 |
return CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, SDValue(Result, 0), |
| 1140 |
getI32Imm(Lo32 >> 16)); |
1140 |
getI32Imm(Lo32 >> 16)); |
| 1141 |
} |
1141 |
} |
| 1142 |
// 2-6) Patterns : {******}{49 zeros}{******} |
1142 |
// 2-6) Patterns : {******}{49 zeros}{******} |
| 1143 |
// {******}{49 ones}{******} |
1143 |
// {******}{49 ones}{******} |
| 1144 |
// If the Imm contains 49 consecutive zeros/ones, it means that a total of 15 |
1144 |
// If the Imm contains 49 consecutive zeros/ones, it means that a total of 15 |
| 1145 |
// bits remain on both sides. Rotate right the Imm to construct an int<16> |
1145 |
// bits remain on both sides. Rotate right the Imm to construct an int<16> |
| 1146 |
// value, use LI for int<16> value and then use RLDICL without mask to rotate |
1146 |
// value, use LI for int<16> value and then use RLDICL without mask to rotate |
| 1147 |
// it back. |
1147 |
// it back. |
| 1148 |
// |
1148 |
// |
| 1149 |
// 1) findContiguousZerosAtLeast(Imm, 49) |
1149 |
// 1) findContiguousZerosAtLeast(Imm, 49) |
| 1150 |
// +------|--zeros-|------+ +---ones--||---15 bit--+ |
1150 |
// +------|--zeros-|------+ +---ones--||---15 bit--+ |
| 1151 |
// |bbbbbb0000000000aaaaaa| -> |0000000000aaaaaabbbbbb| |
1151 |
// |bbbbbb0000000000aaaaaa| -> |0000000000aaaaaabbbbbb| |
| 1152 |
// +----------------------+ +----------------------+ |
1152 |
// +----------------------+ +----------------------+ |
| 1153 |
// 63 0 63 0 |
1153 |
// 63 0 63 0 |
| 1154 |
// |
1154 |
// |
| 1155 |
// 2) findContiguousZerosAtLeast(~Imm, 49) |
1155 |
// 2) findContiguousZerosAtLeast(~Imm, 49) |
| 1156 |
// +------|--ones--|------+ +---ones--||---15 bit--+ |
1156 |
// +------|--ones--|------+ +---ones--||---15 bit--+ |
| 1157 |
// |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb| |
1157 |
// |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb| |
| 1158 |
// +----------------------+ +----------------------+ |
1158 |
// +----------------------+ +----------------------+ |
| 1159 |
// 63 0 63 0 |
1159 |
// 63 0 63 0 |
| 1160 |
if ((Shift = findContiguousZerosAtLeast(Imm, 49)) || |
1160 |
if ((Shift = findContiguousZerosAtLeast(Imm, 49)) || |
| 1161 |
(Shift = findContiguousZerosAtLeast(~Imm, 49))) { |
1161 |
(Shift = findContiguousZerosAtLeast(~Imm, 49))) { |
| 1162 |
uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue(); |
1162 |
uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue(); |
| 1163 |
Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
1163 |
Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
| 1164 |
getI32Imm(RotImm & 0xffff)); |
1164 |
getI32Imm(RotImm & 0xffff)); |
| 1165 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1165 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
| 1166 |
getI32Imm(Shift), getI32Imm(0)); |
1166 |
getI32Imm(Shift), getI32Imm(0)); |
| 1167 |
} |
1167 |
} |
| 1168 |
|
1168 |
|
| 1169 |
// Following patterns use 3 instructions to materialize the Imm. |
1169 |
// Following patterns use 3 instructions to materialize the Imm. |
| 1170 |
InstCnt = 3; |
1170 |
InstCnt = 3; |
| 1171 |
// 3-1) Patterns : {zeros}{ones}{31-bit value}{zeros} |
1171 |
// 3-1) Patterns : {zeros}{ones}{31-bit value}{zeros} |
| 1172 |
// {zeros}{31-bit value}{zeros} |
1172 |
// {zeros}{31-bit value}{zeros} |
| 1173 |
// {zeros}{ones}{31-bit value} |
1173 |
// {zeros}{ones}{31-bit value} |
| 1174 |
// {ones}{31-bit value}{zeros} |
1174 |
// {ones}{31-bit value}{zeros} |
| 1175 |
// We can take advantage of LIS's sign-extension semantics to generate leading |
1175 |
// We can take advantage of LIS's sign-extension semantics to generate leading |
| 1176 |
// ones, add the remaining bits with ORI, and then use RLDIC to mask off the |
1176 |
// ones, add the remaining bits with ORI, and then use RLDIC to mask off the |
| 1177 |
// ones in both sides after rotation. |
1177 |
// ones in both sides after rotation. |
| 1178 |
if ((LZ + FO + TZ) > 32) { |
1178 |
if ((LZ + FO + TZ) > 32) { |
| 1179 |
uint64_t ImmHi16 = (Imm >> (TZ + 16)) & 0xffff; |
1179 |
uint64_t ImmHi16 = (Imm >> (TZ + 16)) & 0xffff; |
| 1180 |
unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; |
1180 |
unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; |
| 1181 |
Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); |
1181 |
Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); |
| 1182 |
Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
1182 |
Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
| 1183 |
getI32Imm((Imm >> TZ) & 0xffff)); |
1183 |
getI32Imm((Imm >> TZ) & 0xffff)); |
| 1184 |
return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), |
1184 |
return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), |
| 1185 |
getI32Imm(TZ), getI32Imm(LZ)); |
1185 |
getI32Imm(TZ), getI32Imm(LZ)); |
| 1186 |
} |
1186 |
} |
| 1187 |
// 3-2) Pattern : {zeros}{31-bit value}{ones} |
1187 |
// 3-2) Pattern : {zeros}{31-bit value}{ones} |
| 1188 |
// Shift right the Imm by (32 - LZ) bits to construct a negative 32 bits |
1188 |
// Shift right the Imm by (32 - LZ) bits to construct a negative 32 bits |
| 1189 |
// value, therefore we can take advantage of LIS's sign-extension semantics, |
1189 |
// value, therefore we can take advantage of LIS's sign-extension semantics, |
| 1190 |
// add the remaining bits with ORI, and then mask them off after rotation. |
1190 |
// add the remaining bits with ORI, and then mask them off after rotation. |
| 1191 |
// This is similar to Pattern 2-3, please refer to the diagram there. |
1191 |
// This is similar to Pattern 2-3, please refer to the diagram there. |
| 1192 |
if ((LZ + TO) > 32) { |
1192 |
if ((LZ + TO) > 32) { |
| 1193 |
// Since the immediates with (LZ > 32) have been handled by previous |
1193 |
// Since the immediates with (LZ > 32) have been handled by previous |
| 1194 |
// patterns, here we have (LZ <= 32) to make sure we will not shift right |
1194 |
// patterns, here we have (LZ <= 32) to make sure we will not shift right |
| 1195 |
// the Imm by a negative value. |
1195 |
// the Imm by a negative value. |
| 1196 |
assert(LZ <= 32 && "Unexpected shift value."); |
1196 |
assert(LZ <= 32 && "Unexpected shift value."); |
| 1197 |
Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, |
1197 |
Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, |
| 1198 |
getI32Imm((Imm >> (48 - LZ)) & 0xffff)); |
1198 |
getI32Imm((Imm >> (48 - LZ)) & 0xffff)); |
| 1199 |
Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
1199 |
Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
| 1200 |
getI32Imm((Imm >> (32 - LZ)) & 0xffff)); |
1200 |
getI32Imm((Imm >> (32 - LZ)) & 0xffff)); |
| 1201 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1201 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
| 1202 |
getI32Imm(32 - LZ), getI32Imm(LZ)); |
1202 |
getI32Imm(32 - LZ), getI32Imm(LZ)); |
| 1203 |
} |
1203 |
} |
| 1204 |
// 3-3) Patterns : {zeros}{ones}{31-bit value}{ones} |
1204 |
// 3-3) Patterns : {zeros}{ones}{31-bit value}{ones} |
| 1205 |
// {ones}{31-bit value}{ones} |
1205 |
// {ones}{31-bit value}{ones} |
| 1206 |
// We can take advantage of LIS's sign-extension semantics to generate leading |
1206 |
// We can take advantage of LIS's sign-extension semantics to generate leading |
| 1207 |
// ones, add the remaining bits with ORI, and then use RLDICL to mask off the |
1207 |
// ones, add the remaining bits with ORI, and then use RLDICL to mask off the |
| 1208 |
// ones in left sides (if required) after rotation. |
1208 |
// ones in left sides (if required) after rotation. |
| 1209 |
// This is similar to Pattern 2-4, please refer to the diagram there. |
1209 |
// This is similar to Pattern 2-4, please refer to the diagram there. |
| 1210 |
if ((LZ + FO + TO) > 32) { |
1210 |
if ((LZ + FO + TO) > 32) { |
| 1211 |
Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, |
1211 |
Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, |
| 1212 |
getI32Imm((Imm >> (TO + 16)) & 0xffff)); |
1212 |
getI32Imm((Imm >> (TO + 16)) & 0xffff)); |
| 1213 |
Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
1213 |
Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
| 1214 |
getI32Imm((Imm >> TO) & 0xffff)); |
1214 |
getI32Imm((Imm >> TO) & 0xffff)); |
| 1215 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1215 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
| 1216 |
getI32Imm(TO), getI32Imm(LZ)); |
1216 |
getI32Imm(TO), getI32Imm(LZ)); |
| 1217 |
} |
1217 |
} |
| 1218 |
// 3-4) Patterns : High word == Low word |
1218 |
// 3-4) Patterns : High word == Low word |
| 1219 |
if (Hi32 == Lo32) { |
1219 |
if (Hi32 == Lo32) { |
| 1220 |
// Handle the first 32 bits. |
1220 |
// Handle the first 32 bits. |
| 1221 |
uint64_t ImmHi16 = (Lo32 >> 16) & 0xffff; |
1221 |
uint64_t ImmHi16 = (Lo32 >> 16) & 0xffff; |
| 1222 |
unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; |
1222 |
unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; |
| 1223 |
Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); |
1223 |
Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); |
| 1224 |
Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
1224 |
Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
| 1225 |
getI32Imm(Lo32 & 0xffff)); |
1225 |
getI32Imm(Lo32 & 0xffff)); |
| 1226 |
// Use rldimi to insert the Low word into High word. |
1226 |
// Use rldimi to insert the Low word into High word. |
| 1227 |
SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32), |
1227 |
SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32), |
| 1228 |
getI32Imm(0)}; |
1228 |
getI32Imm(0)}; |
| 1229 |
return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
1229 |
return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
| 1230 |
} |
1230 |
} |
| 1231 |
// 3-5) Patterns : {******}{33 zeros}{******} |
1231 |
// 3-5) Patterns : {******}{33 zeros}{******} |
| 1232 |
// {******}{33 ones}{******} |
1232 |
// {******}{33 ones}{******} |
| 1233 |
// If the Imm contains 33 consecutive zeros/ones, it means that a total of 31 |
1233 |
// If the Imm contains 33 consecutive zeros/ones, it means that a total of 31 |
| 1234 |
// bits remain on both sides. Rotate right the Imm to construct an int<32> |
1234 |
// bits remain on both sides. Rotate right the Imm to construct an int<32> |
| 1235 |
// value, use LIS + ORI for int<32> value and then use RLDICL without mask to |
1235 |
// value, use LIS + ORI for int<32> value and then use RLDICL without mask to |
| 1236 |
// rotate it back. |
1236 |
// rotate it back. |
| 1237 |
// This is similar to Pattern 2-6, please refer to the diagram there. |
1237 |
// This is similar to Pattern 2-6, please refer to the diagram there. |
| 1238 |
if ((Shift = findContiguousZerosAtLeast(Imm, 33)) || |
1238 |
if ((Shift = findContiguousZerosAtLeast(Imm, 33)) || |
| 1239 |
(Shift = findContiguousZerosAtLeast(~Imm, 33))) { |
1239 |
(Shift = findContiguousZerosAtLeast(~Imm, 33))) { |
| 1240 |
uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue(); |
1240 |
uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue(); |
| 1241 |
uint64_t ImmHi16 = (RotImm >> 16) & 0xffff; |
1241 |
uint64_t ImmHi16 = (RotImm >> 16) & 0xffff; |
| 1242 |
unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; |
1242 |
unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; |
| 1243 |
Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); |
1243 |
Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); |
| 1244 |
Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
1244 |
Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
| 1245 |
getI32Imm(RotImm & 0xffff)); |
1245 |
getI32Imm(RotImm & 0xffff)); |
| 1246 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1246 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
| 1247 |
getI32Imm(Shift), getI32Imm(0)); |
1247 |
getI32Imm(Shift), getI32Imm(0)); |
| 1248 |
} |
1248 |
} |
| 1249 |
|
1249 |
|
| 1250 |
InstCnt = 0; |
1250 |
InstCnt = 0; |
| 1251 |
return nullptr; |
1251 |
return nullptr; |
| 1252 |
} |
1252 |
} |
| 1253 |
|
1253 |
|
| 1254 |
// Try to select instructions to generate a 64 bit immediate using prefix as |
1254 |
// Try to select instructions to generate a 64 bit immediate using prefix as |
| 1255 |
// well as non prefix instructions. The function will return the SDNode |
1255 |
// well as non prefix instructions. The function will return the SDNode |
| 1256 |
// to materialize that constant or it will return nullptr if it does not |
1256 |
// to materialize that constant or it will return nullptr if it does not |
| 1257 |
// find one. The variable InstCnt is set to the number of instructions that |
1257 |
// find one. The variable InstCnt is set to the number of instructions that |
| 1258 |
// were selected. |
1258 |
// were selected. |
| 1259 |
static SDNode *selectI64ImmDirectPrefix(SelectionDAG *CurDAG, const SDLoc &dl, |
1259 |
static SDNode *selectI64ImmDirectPrefix(SelectionDAG *CurDAG, const SDLoc &dl, |
| 1260 |
uint64_t Imm, unsigned &InstCnt) { |
1260 |
uint64_t Imm, unsigned &InstCnt) { |
| 1261 |
unsigned TZ = llvm::countr_zero(Imm); |
1261 |
unsigned TZ = llvm::countr_zero(Imm); |
| 1262 |
unsigned LZ = llvm::countl_zero(Imm); |
1262 |
unsigned LZ = llvm::countl_zero(Imm); |
| 1263 |
unsigned TO = llvm::countr_one(Imm); |
1263 |
unsigned TO = llvm::countr_one(Imm); |
| 1264 |
unsigned FO = llvm::countl_one(LZ == 64 ? 0 : (Imm << LZ)); |
1264 |
unsigned FO = llvm::countl_one(LZ == 64 ? 0 : (Imm << LZ)); |
| 1265 |
unsigned Hi32 = Hi_32(Imm); |
1265 |
unsigned Hi32 = Hi_32(Imm); |
| 1266 |
unsigned Lo32 = Lo_32(Imm); |
1266 |
unsigned Lo32 = Lo_32(Imm); |
| 1267 |
|
1267 |
|
| 1268 |
auto getI32Imm = [CurDAG, dl](unsigned Imm) { |
1268 |
auto getI32Imm = [CurDAG, dl](unsigned Imm) { |
| 1269 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
1269 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
| 1270 |
}; |
1270 |
}; |
| 1271 |
|
1271 |
|
| 1272 |
auto getI64Imm = [CurDAG, dl](uint64_t Imm) { |
1272 |
auto getI64Imm = [CurDAG, dl](uint64_t Imm) { |
| 1273 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i64); |
1273 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i64); |
| 1274 |
}; |
1274 |
}; |
| 1275 |
|
1275 |
|
| 1276 |
// Following patterns use 1 instruction to materialize Imm. |
1276 |
// Following patterns use 1 instruction to materialize Imm. |
| 1277 |
InstCnt = 1; |
1277 |
InstCnt = 1; |
| 1278 |
|
1278 |
|
| 1279 |
// The pli instruction can materialize up to 34 bits directly. |
1279 |
// The pli instruction can materialize up to 34 bits directly. |
| 1280 |
// If a constant fits within 34-bits, emit the pli instruction here directly. |
1280 |
// If a constant fits within 34-bits, emit the pli instruction here directly. |
| 1281 |
if (isInt<34>(Imm)) |
1281 |
if (isInt<34>(Imm)) |
| 1282 |
return CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, |
1282 |
return CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, |
| 1283 |
CurDAG->getTargetConstant(Imm, dl, MVT::i64)); |
1283 |
CurDAG->getTargetConstant(Imm, dl, MVT::i64)); |
| 1284 |
|
1284 |
|
| 1285 |
// Require at least two instructions. |
1285 |
// Require at least two instructions. |
| 1286 |
InstCnt = 2; |
1286 |
InstCnt = 2; |
| 1287 |
SDNode *Result = nullptr; |
1287 |
SDNode *Result = nullptr; |
| 1288 |
// Patterns : {zeros}{ones}{33-bit value}{zeros} |
1288 |
// Patterns : {zeros}{ones}{33-bit value}{zeros} |
| 1289 |
// {zeros}{33-bit value}{zeros} |
1289 |
// {zeros}{33-bit value}{zeros} |
| 1290 |
// {zeros}{ones}{33-bit value} |
1290 |
// {zeros}{ones}{33-bit value} |
| 1291 |
// {ones}{33-bit value}{zeros} |
1291 |
// {ones}{33-bit value}{zeros} |
| 1292 |
// We can take advantage of PLI's sign-extension semantics to generate leading |
1292 |
// We can take advantage of PLI's sign-extension semantics to generate leading |
| 1293 |
// ones, and then use RLDIC to mask off the ones on both sides after rotation. |
1293 |
// ones, and then use RLDIC to mask off the ones on both sides after rotation. |
| 1294 |
if ((LZ + FO + TZ) > 30) { |
1294 |
if ((LZ + FO + TZ) > 30) { |
| 1295 |
APInt SignedInt34 = APInt(34, (Imm >> TZ) & 0x3ffffffff); |
1295 |
APInt SignedInt34 = APInt(34, (Imm >> TZ) & 0x3ffffffff); |
| 1296 |
APInt Extended = SignedInt34.sext(64); |
1296 |
APInt Extended = SignedInt34.sext(64); |
| 1297 |
Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, |
1297 |
Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, |
| 1298 |
getI64Imm(*Extended.getRawData())); |
1298 |
getI64Imm(*Extended.getRawData())); |
| 1299 |
return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), |
1299 |
return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), |
| 1300 |
getI32Imm(TZ), getI32Imm(LZ)); |
1300 |
getI32Imm(TZ), getI32Imm(LZ)); |
| 1301 |
} |
1301 |
} |
| 1302 |
// Pattern : {zeros}{33-bit value}{ones} |
1302 |
// Pattern : {zeros}{33-bit value}{ones} |
| 1303 |
// Shift right the Imm by (30 - LZ) bits to construct a negative 34 bit value, |
1303 |
// Shift right the Imm by (30 - LZ) bits to construct a negative 34 bit value, |
| 1304 |
// therefore we can take advantage of PLI's sign-extension semantics, and then |
1304 |
// therefore we can take advantage of PLI's sign-extension semantics, and then |
| 1305 |
// mask them off after rotation. |
1305 |
// mask them off after rotation. |
| 1306 |
// |
1306 |
// |
| 1307 |
// +--LZ--||-33-bit-||--TO--+ +-------------|--34-bit--+ |
1307 |
// +--LZ--||-33-bit-||--TO--+ +-------------|--34-bit--+ |
| 1308 |
// |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1| |
1308 |
// |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1| |
| 1309 |
// +------------------------+ +------------------------+ |
1309 |
// +------------------------+ +------------------------+ |
| 1310 |
// 63 0 63 0 |
1310 |
// 63 0 63 0 |
| 1311 |
// |
1311 |
// |
| 1312 |
// +----sext-----|--34-bit--+ +clear-|-----------------+ |
1312 |
// +----sext-----|--34-bit--+ +clear-|-----------------+ |
| 1313 |
// |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111| |
1313 |
// |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111| |
| 1314 |
// +------------------------+ +------------------------+ |
1314 |
// +------------------------+ +------------------------+ |
| 1315 |
// 63 0 63 0 |
1315 |
// 63 0 63 0 |
| 1316 |
if ((LZ + TO) > 30) { |
1316 |
if ((LZ + TO) > 30) { |
| 1317 |
APInt SignedInt34 = APInt(34, (Imm >> (30 - LZ)) & 0x3ffffffff); |
1317 |
APInt SignedInt34 = APInt(34, (Imm >> (30 - LZ)) & 0x3ffffffff); |
| 1318 |
APInt Extended = SignedInt34.sext(64); |
1318 |
APInt Extended = SignedInt34.sext(64); |
| 1319 |
Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, |
1319 |
Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, |
| 1320 |
getI64Imm(*Extended.getRawData())); |
1320 |
getI64Imm(*Extended.getRawData())); |
| 1321 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1321 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
| 1322 |
getI32Imm(30 - LZ), getI32Imm(LZ)); |
1322 |
getI32Imm(30 - LZ), getI32Imm(LZ)); |
| 1323 |
} |
1323 |
} |
| 1324 |
// Patterns : {zeros}{ones}{33-bit value}{ones} |
1324 |
// Patterns : {zeros}{ones}{33-bit value}{ones} |
| 1325 |
// {ones}{33-bit value}{ones} |
1325 |
// {ones}{33-bit value}{ones} |
| 1326 |
// Similar to LI we can take advantage of PLI's sign-extension semantics to |
1326 |
// Similar to LI we can take advantage of PLI's sign-extension semantics to |
| 1327 |
// generate leading ones, and then use RLDICL to mask off the ones in left |
1327 |
// generate leading ones, and then use RLDICL to mask off the ones in left |
| 1328 |
// sides (if required) after rotation. |
1328 |
// sides (if required) after rotation. |
| 1329 |
if ((LZ + FO + TO) > 30) { |
1329 |
if ((LZ + FO + TO) > 30) { |
| 1330 |
APInt SignedInt34 = APInt(34, (Imm >> TO) & 0x3ffffffff); |
1330 |
APInt SignedInt34 = APInt(34, (Imm >> TO) & 0x3ffffffff); |
| 1331 |
APInt Extended = SignedInt34.sext(64); |
1331 |
APInt Extended = SignedInt34.sext(64); |
| 1332 |
Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, |
1332 |
Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, |
| 1333 |
getI64Imm(*Extended.getRawData())); |
1333 |
getI64Imm(*Extended.getRawData())); |
| 1334 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1334 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
| 1335 |
getI32Imm(TO), getI32Imm(LZ)); |
1335 |
getI32Imm(TO), getI32Imm(LZ)); |
| 1336 |
} |
1336 |
} |
| 1337 |
// Patterns : {******}{31 zeros}{******} |
1337 |
// Patterns : {******}{31 zeros}{******} |
| 1338 |
// : {******}{31 ones}{******} |
1338 |
// : {******}{31 ones}{******} |
| 1339 |
// If Imm contains 31 consecutive zeros/ones then the remaining bit count |
1339 |
// If Imm contains 31 consecutive zeros/ones then the remaining bit count |
| 1340 |
// is 33. Rotate right the Imm to construct a int<33> value, we can use PLI |
1340 |
// is 33. Rotate right the Imm to construct a int<33> value, we can use PLI |
| 1341 |
// for the int<33> value and then use RLDICL without a mask to rotate it back. |
1341 |
// for the int<33> value and then use RLDICL without a mask to rotate it back. |
| 1342 |
// |
1342 |
// |
| 1343 |
// +------|--ones--|------+ +---ones--||---33 bit--+ |
1343 |
// +------|--ones--|------+ +---ones--||---33 bit--+ |
| 1344 |
// |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb| |
1344 |
// |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb| |
| 1345 |
// +----------------------+ +----------------------+ |
1345 |
// +----------------------+ +----------------------+ |
| 1346 |
// 63 0 63 0 |
1346 |
// 63 0 63 0 |
| 1347 |
for (unsigned Shift = 0; Shift < 63; ++Shift) { |
1347 |
for (unsigned Shift = 0; Shift < 63; ++Shift) { |
| 1348 |
uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue(); |
1348 |
uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue(); |
| 1349 |
if (isInt<34>(RotImm)) { |
1349 |
if (isInt<34>(RotImm)) { |
| 1350 |
Result = |
1350 |
Result = |
| 1351 |
CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(RotImm)); |
1351 |
CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(RotImm)); |
| 1352 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
1352 |
return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
| 1353 |
SDValue(Result, 0), getI32Imm(Shift), |
1353 |
SDValue(Result, 0), getI32Imm(Shift), |
| 1354 |
getI32Imm(0)); |
1354 |
getI32Imm(0)); |
| 1355 |
} |
1355 |
} |
| 1356 |
} |
1356 |
} |
| 1357 |
|
1357 |
|
| 1358 |
// Patterns : High word == Low word |
1358 |
// Patterns : High word == Low word |
| 1359 |
// This is basically a splat of a 32 bit immediate. |
1359 |
// This is basically a splat of a 32 bit immediate. |
| 1360 |
if (Hi32 == Lo32) { |
1360 |
if (Hi32 == Lo32) { |
| 1361 |
Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32)); |
1361 |
Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32)); |
| 1362 |
SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32), |
1362 |
SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32), |
| 1363 |
getI32Imm(0)}; |
1363 |
getI32Imm(0)}; |
| 1364 |
return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
1364 |
return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
| 1365 |
} |
1365 |
} |
| 1366 |
|
1366 |
|
| 1367 |
InstCnt = 3; |
1367 |
InstCnt = 3; |
| 1368 |
// Catch-all |
1368 |
// Catch-all |
| 1369 |
// This pattern can form any 64 bit immediate in 3 instructions. |
1369 |
// This pattern can form any 64 bit immediate in 3 instructions. |
| 1370 |
SDNode *ResultHi = |
1370 |
SDNode *ResultHi = |
| 1371 |
CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32)); |
1371 |
CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32)); |
| 1372 |
SDNode *ResultLo = |
1372 |
SDNode *ResultLo = |
| 1373 |
CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Lo32)); |
1373 |
CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Lo32)); |
| 1374 |
SDValue Ops[] = {SDValue(ResultLo, 0), SDValue(ResultHi, 0), getI32Imm(32), |
1374 |
SDValue Ops[] = {SDValue(ResultLo, 0), SDValue(ResultHi, 0), getI32Imm(32), |
| 1375 |
getI32Imm(0)}; |
1375 |
getI32Imm(0)}; |
| 1376 |
return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
1376 |
return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
| 1377 |
} |
1377 |
} |
| 1378 |
|
1378 |
|
| 1379 |
static SDNode *selectI64Imm(SelectionDAG *CurDAG, const SDLoc &dl, uint64_t Imm, |
1379 |
static SDNode *selectI64Imm(SelectionDAG *CurDAG, const SDLoc &dl, uint64_t Imm, |
| 1380 |
unsigned *InstCnt = nullptr) { |
1380 |
unsigned *InstCnt = nullptr) { |
| 1381 |
unsigned InstCntDirect = 0; |
1381 |
unsigned InstCntDirect = 0; |
| 1382 |
// No more than 3 instructions are used if we can select the i64 immediate |
1382 |
// No more than 3 instructions are used if we can select the i64 immediate |
| 1383 |
// directly. |
1383 |
// directly. |
| 1384 |
SDNode *Result = selectI64ImmDirect(CurDAG, dl, Imm, InstCntDirect); |
1384 |
SDNode *Result = selectI64ImmDirect(CurDAG, dl, Imm, InstCntDirect); |
| 1385 |
|
1385 |
|
| 1386 |
const PPCSubtarget &Subtarget = |
1386 |
const PPCSubtarget &Subtarget = |
| 1387 |
CurDAG->getMachineFunction().getSubtarget(); |
1387 |
CurDAG->getMachineFunction().getSubtarget(); |
| 1388 |
|
1388 |
|
| 1389 |
// If we have prefixed instructions and there is a chance we can |
1389 |
// If we have prefixed instructions and there is a chance we can |
| 1390 |
// materialize the constant with fewer prefixed instructions than |
1390 |
// materialize the constant with fewer prefixed instructions than |
| 1391 |
// non-prefixed, try that. |
1391 |
// non-prefixed, try that. |
| 1392 |
if (Subtarget.hasPrefixInstrs() && InstCntDirect != 1) { |
1392 |
if (Subtarget.hasPrefixInstrs() && InstCntDirect != 1) { |
| 1393 |
unsigned InstCntDirectP = 0; |
1393 |
unsigned InstCntDirectP = 0; |
| 1394 |
SDNode *ResultP = selectI64ImmDirectPrefix(CurDAG, dl, Imm, InstCntDirectP); |
1394 |
SDNode *ResultP = selectI64ImmDirectPrefix(CurDAG, dl, Imm, InstCntDirectP); |
| 1395 |
// Use the prefix case in either of two cases: |
1395 |
// Use the prefix case in either of two cases: |
| 1396 |
// 1) We have no result from the non-prefix case to use. |
1396 |
// 1) We have no result from the non-prefix case to use. |
| 1397 |
// 2) The non-prefix case uses more instructions than the prefix case. |
1397 |
// 2) The non-prefix case uses more instructions than the prefix case. |
| 1398 |
// If the prefix and non-prefix cases use the same number of instructions |
1398 |
// If the prefix and non-prefix cases use the same number of instructions |
| 1399 |
// we will prefer the non-prefix case. |
1399 |
// we will prefer the non-prefix case. |
| 1400 |
if (ResultP && (!Result || InstCntDirectP < InstCntDirect)) { |
1400 |
if (ResultP && (!Result || InstCntDirectP < InstCntDirect)) { |
| 1401 |
if (InstCnt) |
1401 |
if (InstCnt) |
| 1402 |
*InstCnt = InstCntDirectP; |
1402 |
*InstCnt = InstCntDirectP; |
| 1403 |
return ResultP; |
1403 |
return ResultP; |
| 1404 |
} |
1404 |
} |
| 1405 |
} |
1405 |
} |
| 1406 |
|
1406 |
|
| 1407 |
if (Result) { |
1407 |
if (Result) { |
| 1408 |
if (InstCnt) |
1408 |
if (InstCnt) |
| 1409 |
*InstCnt = InstCntDirect; |
1409 |
*InstCnt = InstCntDirect; |
| 1410 |
return Result; |
1410 |
return Result; |
| 1411 |
} |
1411 |
} |
| 1412 |
auto getI32Imm = [CurDAG, dl](unsigned Imm) { |
1412 |
auto getI32Imm = [CurDAG, dl](unsigned Imm) { |
| 1413 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
1413 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
| 1414 |
}; |
1414 |
}; |
| 1415 |
|
1415 |
|
| 1416 |
uint32_t Hi16OfLo32 = (Lo_32(Imm) >> 16) & 0xffff; |
1416 |
uint32_t Hi16OfLo32 = (Lo_32(Imm) >> 16) & 0xffff; |
| 1417 |
uint32_t Lo16OfLo32 = Lo_32(Imm) & 0xffff; |
1417 |
uint32_t Lo16OfLo32 = Lo_32(Imm) & 0xffff; |
| 1418 |
|
1418 |
|
| 1419 |
// Try to use 4 instructions to materialize the immediate which is "almost" a |
1419 |
// Try to use 4 instructions to materialize the immediate which is "almost" a |
| 1420 |
// splat of a 32 bit immediate. |
1420 |
// splat of a 32 bit immediate. |
| 1421 |
if (Hi16OfLo32 && Lo16OfLo32) { |
1421 |
if (Hi16OfLo32 && Lo16OfLo32) { |
| 1422 |
uint32_t Hi16OfHi32 = (Hi_32(Imm) >> 16) & 0xffff; |
1422 |
uint32_t Hi16OfHi32 = (Hi_32(Imm) >> 16) & 0xffff; |
| 1423 |
uint32_t Lo16OfHi32 = Hi_32(Imm) & 0xffff; |
1423 |
uint32_t Lo16OfHi32 = Hi_32(Imm) & 0xffff; |
| 1424 |
bool IsSelected = false; |
1424 |
bool IsSelected = false; |
| 1425 |
|
1425 |
|
| 1426 |
auto getSplat = [CurDAG, dl, getI32Imm](uint32_t Hi16, uint32_t Lo16) { |
1426 |
auto getSplat = [CurDAG, dl, getI32Imm](uint32_t Hi16, uint32_t Lo16) { |
| 1427 |
SDNode *Result = |
1427 |
SDNode *Result = |
| 1428 |
CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, getI32Imm(Hi16)); |
1428 |
CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, getI32Imm(Hi16)); |
| 1429 |
Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, |
1429 |
Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, |
| 1430 |
SDValue(Result, 0), getI32Imm(Lo16)); |
1430 |
SDValue(Result, 0), getI32Imm(Lo16)); |
| 1431 |
SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32), |
1431 |
SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32), |
| 1432 |
getI32Imm(0)}; |
1432 |
getI32Imm(0)}; |
| 1433 |
return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
1433 |
return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
| 1434 |
}; |
1434 |
}; |
| 1435 |
|
1435 |
|
| 1436 |
if (Hi16OfHi32 == Lo16OfHi32 && Lo16OfHi32 == Lo16OfLo32) { |
1436 |
if (Hi16OfHi32 == Lo16OfHi32 && Lo16OfHi32 == Lo16OfLo32) { |
| 1437 |
IsSelected = true; |
1437 |
IsSelected = true; |
| 1438 |
Result = getSplat(Hi16OfLo32, Lo16OfLo32); |
1438 |
Result = getSplat(Hi16OfLo32, Lo16OfLo32); |
| 1439 |
// Modify Hi16OfHi32. |
1439 |
// Modify Hi16OfHi32. |
| 1440 |
SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(48), |
1440 |
SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(48), |
| 1441 |
getI32Imm(0)}; |
1441 |
getI32Imm(0)}; |
| 1442 |
Result = CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
1442 |
Result = CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
| 1443 |
} else if (Hi16OfHi32 == Hi16OfLo32 && Hi16OfLo32 == Lo16OfLo32) { |
1443 |
} else if (Hi16OfHi32 == Hi16OfLo32 && Hi16OfLo32 == Lo16OfLo32) { |
| 1444 |
IsSelected = true; |
1444 |
IsSelected = true; |
| 1445 |
Result = getSplat(Hi16OfHi32, Lo16OfHi32); |
1445 |
Result = getSplat(Hi16OfHi32, Lo16OfHi32); |
| 1446 |
// Modify Lo16OfLo32. |
1446 |
// Modify Lo16OfLo32. |
| 1447 |
SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(16), |
1447 |
SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(16), |
| 1448 |
getI32Imm(16), getI32Imm(31)}; |
1448 |
getI32Imm(16), getI32Imm(31)}; |
| 1449 |
Result = CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, Ops); |
1449 |
Result = CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, Ops); |
| 1450 |
} else if (Lo16OfHi32 == Lo16OfLo32 && Hi16OfLo32 == Lo16OfLo32) { |
1450 |
} else if (Lo16OfHi32 == Lo16OfLo32 && Hi16OfLo32 == Lo16OfLo32) { |
| 1451 |
IsSelected = true; |
1451 |
IsSelected = true; |
| 1452 |
Result = getSplat(Hi16OfHi32, Lo16OfHi32); |
1452 |
Result = getSplat(Hi16OfHi32, Lo16OfHi32); |
| 1453 |
// Modify Hi16OfLo32. |
1453 |
// Modify Hi16OfLo32. |
| 1454 |
SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(16), |
1454 |
SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(16), |
| 1455 |
getI32Imm(0), getI32Imm(15)}; |
1455 |
getI32Imm(0), getI32Imm(15)}; |
| 1456 |
Result = CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, Ops); |
1456 |
Result = CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, Ops); |
| 1457 |
} |
1457 |
} |
| 1458 |
if (IsSelected == true) { |
1458 |
if (IsSelected == true) { |
| 1459 |
if (InstCnt) |
1459 |
if (InstCnt) |
| 1460 |
*InstCnt = 4; |
1460 |
*InstCnt = 4; |
| 1461 |
return Result; |
1461 |
return Result; |
| 1462 |
} |
1462 |
} |
| 1463 |
} |
1463 |
} |
| 1464 |
|
1464 |
|
| 1465 |
// Handle the upper 32 bit value. |
1465 |
// Handle the upper 32 bit value. |
| 1466 |
Result = |
1466 |
Result = |
| 1467 |
selectI64ImmDirect(CurDAG, dl, Imm & 0xffffffff00000000, InstCntDirect); |
1467 |
selectI64ImmDirect(CurDAG, dl, Imm & 0xffffffff00000000, InstCntDirect); |
| 1468 |
// Add in the last bits as required. |
1468 |
// Add in the last bits as required. |
| 1469 |
if (Hi16OfLo32) { |
1469 |
if (Hi16OfLo32) { |
| 1470 |
Result = CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, |
1470 |
Result = CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, |
| 1471 |
SDValue(Result, 0), getI32Imm(Hi16OfLo32)); |
1471 |
SDValue(Result, 0), getI32Imm(Hi16OfLo32)); |
| 1472 |
++InstCntDirect; |
1472 |
++InstCntDirect; |
| 1473 |
} |
1473 |
} |
| 1474 |
if (Lo16OfLo32) { |
1474 |
if (Lo16OfLo32) { |
| 1475 |
Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
1475 |
Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
| 1476 |
getI32Imm(Lo16OfLo32)); |
1476 |
getI32Imm(Lo16OfLo32)); |
| 1477 |
++InstCntDirect; |
1477 |
++InstCntDirect; |
| 1478 |
} |
1478 |
} |
| 1479 |
if (InstCnt) |
1479 |
if (InstCnt) |
| 1480 |
*InstCnt = InstCntDirect; |
1480 |
*InstCnt = InstCntDirect; |
| 1481 |
return Result; |
1481 |
return Result; |
| 1482 |
} |
1482 |
} |
| 1483 |
|
1483 |
|
| 1484 |
// Select a 64-bit constant. |
1484 |
// Select a 64-bit constant. |
| 1485 |
static SDNode *selectI64Imm(SelectionDAG *CurDAG, SDNode *N) { |
1485 |
static SDNode *selectI64Imm(SelectionDAG *CurDAG, SDNode *N) { |
| 1486 |
SDLoc dl(N); |
1486 |
SDLoc dl(N); |
| 1487 |
|
1487 |
|
| 1488 |
// Get 64 bit value. |
1488 |
// Get 64 bit value. |
| 1489 |
int64_t Imm = cast(N)->getZExtValue(); |
1489 |
int64_t Imm = cast(N)->getZExtValue(); |
| 1490 |
if (unsigned MinSize = allUsesTruncate(CurDAG, N)) { |
1490 |
if (unsigned MinSize = allUsesTruncate(CurDAG, N)) { |
| 1491 |
uint64_t SextImm = SignExtend64(Imm, MinSize); |
1491 |
uint64_t SextImm = SignExtend64(Imm, MinSize); |
| 1492 |
SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64); |
1492 |
SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64); |
| 1493 |
if (isInt<16>(SextImm)) |
1493 |
if (isInt<16>(SextImm)) |
| 1494 |
return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm); |
1494 |
return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm); |
| 1495 |
} |
1495 |
} |
| 1496 |
return selectI64Imm(CurDAG, dl, Imm); |
1496 |
return selectI64Imm(CurDAG, dl, Imm); |
| 1497 |
} |
1497 |
} |
| 1498 |
|
1498 |
|
| 1499 |
namespace { |
1499 |
namespace { |
| 1500 |
|
1500 |
|
| 1501 |
class BitPermutationSelector { |
1501 |
class BitPermutationSelector { |
| 1502 |
struct ValueBit { |
1502 |
struct ValueBit { |
| 1503 |
SDValue V; |
1503 |
SDValue V; |
| 1504 |
|
1504 |
|
| 1505 |
// The bit number in the value, using a convention where bit 0 is the |
1505 |
// The bit number in the value, using a convention where bit 0 is the |
| 1506 |
// lowest-order bit. |
1506 |
// lowest-order bit. |
| 1507 |
unsigned Idx; |
1507 |
unsigned Idx; |
| 1508 |
|
1508 |
|
| 1509 |
// ConstZero means a bit we need to mask off. |
1509 |
// ConstZero means a bit we need to mask off. |
| 1510 |
// Variable is a bit comes from an input variable. |
1510 |
// Variable is a bit comes from an input variable. |
| 1511 |
// VariableKnownToBeZero is also a bit comes from an input variable, |
1511 |
// VariableKnownToBeZero is also a bit comes from an input variable, |
| 1512 |
// but it is known to be already zero. So we do not need to mask them. |
1512 |
// but it is known to be already zero. So we do not need to mask them. |
| 1513 |
enum Kind { |
1513 |
enum Kind { |
| 1514 |
ConstZero, |
1514 |
ConstZero, |
| 1515 |
Variable, |
1515 |
Variable, |
| 1516 |
VariableKnownToBeZero |
1516 |
VariableKnownToBeZero |
| 1517 |
} K; |
1517 |
} K; |
| 1518 |
|
1518 |
|
| 1519 |
ValueBit(SDValue V, unsigned I, Kind K = Variable) |
1519 |
ValueBit(SDValue V, unsigned I, Kind K = Variable) |
| 1520 |
: V(V), Idx(I), K(K) {} |
1520 |
: V(V), Idx(I), K(K) {} |
| 1521 |
ValueBit(Kind K = Variable) : Idx(UINT32_MAX), K(K) {} |
1521 |
ValueBit(Kind K = Variable) : Idx(UINT32_MAX), K(K) {} |
| 1522 |
|
1522 |
|
| 1523 |
bool isZero() const { |
1523 |
bool isZero() const { |
| 1524 |
return K == ConstZero || K == VariableKnownToBeZero; |
1524 |
return K == ConstZero || K == VariableKnownToBeZero; |
| 1525 |
} |
1525 |
} |
| 1526 |
|
1526 |
|
| 1527 |
bool hasValue() const { |
1527 |
bool hasValue() const { |
| 1528 |
return K == Variable || K == VariableKnownToBeZero; |
1528 |
return K == Variable || K == VariableKnownToBeZero; |
| 1529 |
} |
1529 |
} |
| 1530 |
|
1530 |
|
| 1531 |
SDValue getValue() const { |
1531 |
SDValue getValue() const { |
| 1532 |
assert(hasValue() && "Cannot get the value of a constant bit"); |
1532 |
assert(hasValue() && "Cannot get the value of a constant bit"); |
| 1533 |
return V; |
1533 |
return V; |
| 1534 |
} |
1534 |
} |
| 1535 |
|
1535 |
|
| 1536 |
unsigned getValueBitIndex() const { |
1536 |
unsigned getValueBitIndex() const { |
| 1537 |
assert(hasValue() && "Cannot get the value bit index of a constant bit"); |
1537 |
assert(hasValue() && "Cannot get the value bit index of a constant bit"); |
| 1538 |
return Idx; |
1538 |
return Idx; |
| 1539 |
} |
1539 |
} |
| 1540 |
}; |
1540 |
}; |
| 1541 |
|
1541 |
|
| 1542 |
// A bit group has the same underlying value and the same rotate factor. |
1542 |
// A bit group has the same underlying value and the same rotate factor. |
| 1543 |
struct BitGroup { |
1543 |
struct BitGroup { |
| 1544 |
SDValue V; |
1544 |
SDValue V; |
| 1545 |
unsigned RLAmt; |
1545 |
unsigned RLAmt; |
| 1546 |
unsigned StartIdx, EndIdx; |
1546 |
unsigned StartIdx, EndIdx; |
| 1547 |
|
1547 |
|
| 1548 |
// This rotation amount assumes that the lower 32 bits of the quantity are |
1548 |
// This rotation amount assumes that the lower 32 bits of the quantity are |
| 1549 |
// replicated in the high 32 bits by the rotation operator (which is done |
1549 |
// replicated in the high 32 bits by the rotation operator (which is done |
| 1550 |
// by rlwinm and friends in 64-bit mode). |
1550 |
// by rlwinm and friends in 64-bit mode). |
| 1551 |
bool Repl32; |
1551 |
bool Repl32; |
| 1552 |
// Did converting to Repl32 == true change the rotation factor? If it did, |
1552 |
// Did converting to Repl32 == true change the rotation factor? If it did, |
| 1553 |
// it decreased it by 32. |
1553 |
// it decreased it by 32. |
| 1554 |
bool Repl32CR; |
1554 |
bool Repl32CR; |
| 1555 |
// Was this group coalesced after setting Repl32 to true? |
1555 |
// Was this group coalesced after setting Repl32 to true? |
| 1556 |
bool Repl32Coalesced; |
1556 |
bool Repl32Coalesced; |
| 1557 |
|
1557 |
|
| 1558 |
BitGroup(SDValue V, unsigned R, unsigned S, unsigned E) |
1558 |
BitGroup(SDValue V, unsigned R, unsigned S, unsigned E) |
| 1559 |
: V(V), RLAmt(R), StartIdx(S), EndIdx(E), Repl32(false), Repl32CR(false), |
1559 |
: V(V), RLAmt(R), StartIdx(S), EndIdx(E), Repl32(false), Repl32CR(false), |
| 1560 |
Repl32Coalesced(false) { |
1560 |
Repl32Coalesced(false) { |
| 1561 |
LLVM_DEBUG(dbgs() << "\tbit group for " << V.getNode() << " RLAmt = " << R |
1561 |
LLVM_DEBUG(dbgs() << "\tbit group for " << V.getNode() << " RLAmt = " << R |
| 1562 |
<< " [" << S << ", " << E << "]\n"); |
1562 |
<< " [" << S << ", " << E << "]\n"); |
| 1563 |
} |
1563 |
} |
| 1564 |
}; |
1564 |
}; |
| 1565 |
|
1565 |
|
| 1566 |
// Information on each (Value, RLAmt) pair (like the number of groups |
1566 |
// Information on each (Value, RLAmt) pair (like the number of groups |
| 1567 |
// associated with each) used to choose the lowering method. |
1567 |
// associated with each) used to choose the lowering method. |
| 1568 |
struct ValueRotInfo { |
1568 |
struct ValueRotInfo { |
| 1569 |
SDValue V; |
1569 |
SDValue V; |
| 1570 |
unsigned RLAmt = std::numeric_limits::max(); |
1570 |
unsigned RLAmt = std::numeric_limits::max(); |
| 1571 |
unsigned NumGroups = 0; |
1571 |
unsigned NumGroups = 0; |
| 1572 |
unsigned FirstGroupStartIdx = std::numeric_limits::max(); |
1572 |
unsigned FirstGroupStartIdx = std::numeric_limits::max(); |
| 1573 |
bool Repl32 = false; |
1573 |
bool Repl32 = false; |
| 1574 |
|
1574 |
|
| 1575 |
ValueRotInfo() = default; |
1575 |
ValueRotInfo() = default; |
| 1576 |
|
1576 |
|
| 1577 |
// For sorting (in reverse order) by NumGroups, and then by |
1577 |
// For sorting (in reverse order) by NumGroups, and then by |
| 1578 |
// FirstGroupStartIdx. |
1578 |
// FirstGroupStartIdx. |
| 1579 |
bool operator < (const ValueRotInfo &Other) const { |
1579 |
bool operator < (const ValueRotInfo &Other) const { |
| 1580 |
// We need to sort so that the non-Repl32 come first because, when we're |
1580 |
// We need to sort so that the non-Repl32 come first because, when we're |
| 1581 |
// doing masking, the Repl32 bit groups might be subsumed into the 64-bit |
1581 |
// doing masking, the Repl32 bit groups might be subsumed into the 64-bit |
| 1582 |
// masking operation. |
1582 |
// masking operation. |
| 1583 |
if (Repl32 < Other.Repl32) |
1583 |
if (Repl32 < Other.Repl32) |
| 1584 |
return true; |
1584 |
return true; |
| 1585 |
else if (Repl32 > Other.Repl32) |
1585 |
else if (Repl32 > Other.Repl32) |
| 1586 |
return false; |
1586 |
return false; |
| 1587 |
else if (NumGroups > Other.NumGroups) |
1587 |
else if (NumGroups > Other.NumGroups) |
| 1588 |
return true; |
1588 |
return true; |
| 1589 |
else if (NumGroups < Other.NumGroups) |
1589 |
else if (NumGroups < Other.NumGroups) |
| 1590 |
return false; |
1590 |
return false; |
| 1591 |
else if (RLAmt == 0 && Other.RLAmt != 0) |
1591 |
else if (RLAmt == 0 && Other.RLAmt != 0) |
| 1592 |
return true; |
1592 |
return true; |
| 1593 |
else if (RLAmt != 0 && Other.RLAmt == 0) |
1593 |
else if (RLAmt != 0 && Other.RLAmt == 0) |
| 1594 |
return false; |
1594 |
return false; |
| 1595 |
else if (FirstGroupStartIdx < Other.FirstGroupStartIdx) |
1595 |
else if (FirstGroupStartIdx < Other.FirstGroupStartIdx) |
| 1596 |
return true; |
1596 |
return true; |
| 1597 |
return false; |
1597 |
return false; |
| 1598 |
} |
1598 |
} |
| 1599 |
}; |
1599 |
}; |
| 1600 |
|
1600 |
|
| 1601 |
using ValueBitsMemoizedValue = std::pair>; |
1601 |
using ValueBitsMemoizedValue = std::pair>; |
| 1602 |
using ValueBitsMemoizer = |
1602 |
using ValueBitsMemoizer = |
| 1603 |
DenseMap>; |
1603 |
DenseMap>; |
| 1604 |
ValueBitsMemoizer Memoizer; |
1604 |
ValueBitsMemoizer Memoizer; |
| 1605 |
|
1605 |
|
| 1606 |
// Return a pair of bool and a SmallVector pointer to a memoization entry. |
1606 |
// Return a pair of bool and a SmallVector pointer to a memoization entry. |
| 1607 |
// The bool is true if something interesting was deduced, otherwise if we're |
1607 |
// The bool is true if something interesting was deduced, otherwise if we're |
| 1608 |
// providing only a generic representation of V (or something else likewise |
1608 |
// providing only a generic representation of V (or something else likewise |
| 1609 |
// uninteresting for instruction selection) through the SmallVector. |
1609 |
// uninteresting for instruction selection) through the SmallVector. |
| 1610 |
std::pair *> getValueBits(SDValue V, |
1610 |
std::pair *> getValueBits(SDValue V, |
| 1611 |
unsigned NumBits) { |
1611 |
unsigned NumBits) { |
| 1612 |
auto &ValueEntry = Memoizer[V]; |
1612 |
auto &ValueEntry = Memoizer[V]; |
| 1613 |
if (ValueEntry) |
1613 |
if (ValueEntry) |
| 1614 |
return std::make_pair(ValueEntry->first, &ValueEntry->second); |
1614 |
return std::make_pair(ValueEntry->first, &ValueEntry->second); |
| 1615 |
ValueEntry.reset(new ValueBitsMemoizedValue()); |
1615 |
ValueEntry.reset(new ValueBitsMemoizedValue()); |
| 1616 |
bool &Interesting = ValueEntry->first; |
1616 |
bool &Interesting = ValueEntry->first; |
| 1617 |
SmallVector &Bits = ValueEntry->second; |
1617 |
SmallVector &Bits = ValueEntry->second; |
| 1618 |
Bits.resize(NumBits); |
1618 |
Bits.resize(NumBits); |
| 1619 |
|
1619 |
|
| 1620 |
switch (V.getOpcode()) { |
1620 |
switch (V.getOpcode()) { |
| 1621 |
default: break; |
1621 |
default: break; |
| 1622 |
case ISD::ROTL: |
1622 |
case ISD::ROTL: |
| 1623 |
if (isa(V.getOperand(1))) { |
1623 |
if (isa(V.getOperand(1))) { |
| 1624 |
unsigned RotAmt = V.getConstantOperandVal(1); |
1624 |
unsigned RotAmt = V.getConstantOperandVal(1); |
| 1625 |
|
1625 |
|
| 1626 |
const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; |
1626 |
const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; |
| 1627 |
|
1627 |
|
| 1628 |
for (unsigned i = 0; i < NumBits; ++i) |
1628 |
for (unsigned i = 0; i < NumBits; ++i) |
| 1629 |
Bits[i] = LHSBits[i < RotAmt ? i + (NumBits - RotAmt) : i - RotAmt]; |
1629 |
Bits[i] = LHSBits[i < RotAmt ? i + (NumBits - RotAmt) : i - RotAmt]; |
| 1630 |
|
1630 |
|
| 1631 |
return std::make_pair(Interesting = true, &Bits); |
1631 |
return std::make_pair(Interesting = true, &Bits); |
| 1632 |
} |
1632 |
} |
| 1633 |
break; |
1633 |
break; |
| 1634 |
case ISD::SHL: |
1634 |
case ISD::SHL: |
| 1635 |
case PPCISD::SHL: |
1635 |
case PPCISD::SHL: |
| 1636 |
if (isa(V.getOperand(1))) { |
1636 |
if (isa(V.getOperand(1))) { |
| 1637 |
unsigned ShiftAmt = V.getConstantOperandVal(1); |
1637 |
unsigned ShiftAmt = V.getConstantOperandVal(1); |
| 1638 |
|
1638 |
|
| 1639 |
const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; |
1639 |
const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; |
| 1640 |
|
1640 |
|
| 1641 |
for (unsigned i = ShiftAmt; i < NumBits; ++i) |
1641 |
for (unsigned i = ShiftAmt; i < NumBits; ++i) |
| 1642 |
Bits[i] = LHSBits[i - ShiftAmt]; |
1642 |
Bits[i] = LHSBits[i - ShiftAmt]; |
| 1643 |
|
1643 |
|
| 1644 |
for (unsigned i = 0; i < ShiftAmt; ++i) |
1644 |
for (unsigned i = 0; i < ShiftAmt; ++i) |
| 1645 |
Bits[i] = ValueBit(ValueBit::ConstZero); |
1645 |
Bits[i] = ValueBit(ValueBit::ConstZero); |
| 1646 |
|
1646 |
|
| 1647 |
return std::make_pair(Interesting = true, &Bits); |
1647 |
return std::make_pair(Interesting = true, &Bits); |
| 1648 |
} |
1648 |
} |
| 1649 |
break; |
1649 |
break; |
| 1650 |
case ISD::SRL: |
1650 |
case ISD::SRL: |
| 1651 |
case PPCISD::SRL: |
1651 |
case PPCISD::SRL: |
| 1652 |
if (isa(V.getOperand(1))) { |
1652 |
if (isa(V.getOperand(1))) { |
| 1653 |
unsigned ShiftAmt = V.getConstantOperandVal(1); |
1653 |
unsigned ShiftAmt = V.getConstantOperandVal(1); |
| 1654 |
|
1654 |
|
| 1655 |
const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; |
1655 |
const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; |
| 1656 |
|
1656 |
|
| 1657 |
for (unsigned i = 0; i < NumBits - ShiftAmt; ++i) |
1657 |
for (unsigned i = 0; i < NumBits - ShiftAmt; ++i) |
| 1658 |
Bits[i] = LHSBits[i + ShiftAmt]; |
1658 |
Bits[i] = LHSBits[i + ShiftAmt]; |
| 1659 |
|
1659 |
|
| 1660 |
for (unsigned i = NumBits - ShiftAmt; i < NumBits; ++i) |
1660 |
for (unsigned i = NumBits - ShiftAmt; i < NumBits; ++i) |
| 1661 |
Bits[i] = ValueBit(ValueBit::ConstZero); |
1661 |
Bits[i] = ValueBit(ValueBit::ConstZero); |
| 1662 |
|
1662 |
|
| 1663 |
return std::make_pair(Interesting = true, &Bits); |
1663 |
return std::make_pair(Interesting = true, &Bits); |
| 1664 |
} |
1664 |
} |
| 1665 |
break; |
1665 |
break; |
| 1666 |
case ISD::AND: |
1666 |
case ISD::AND: |
| 1667 |
if (isa(V.getOperand(1))) { |
1667 |
if (isa(V.getOperand(1))) { |
| 1668 |
uint64_t Mask = V.getConstantOperandVal(1); |
1668 |
uint64_t Mask = V.getConstantOperandVal(1); |
| 1669 |
|
1669 |
|
| 1670 |
const SmallVector *LHSBits; |
1670 |
const SmallVector *LHSBits; |
| 1671 |
// Mark this as interesting, only if the LHS was also interesting. This |
1671 |
// Mark this as interesting, only if the LHS was also interesting. This |
| 1672 |
// prevents the overall procedure from matching a single immediate 'and' |
1672 |
// prevents the overall procedure from matching a single immediate 'and' |
| 1673 |
// (which is non-optimal because such an and might be folded with other |
1673 |
// (which is non-optimal because such an and might be folded with other |
| 1674 |
// things if we don't select it here). |
1674 |
// things if we don't select it here). |
| 1675 |
std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), NumBits); |
1675 |
std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), NumBits); |
| 1676 |
|
1676 |
|
| 1677 |
for (unsigned i = 0; i < NumBits; ++i) |
1677 |
for (unsigned i = 0; i < NumBits; ++i) |
| 1678 |
if (((Mask >> i) & 1) == 1) |
1678 |
if (((Mask >> i) & 1) == 1) |
| 1679 |
Bits[i] = (*LHSBits)[i]; |
1679 |
Bits[i] = (*LHSBits)[i]; |
| 1680 |
else { |
1680 |
else { |
| 1681 |
// AND instruction masks this bit. If the input is already zero, |
1681 |
// AND instruction masks this bit. If the input is already zero, |
| 1682 |
// we have nothing to do here. Otherwise, make the bit ConstZero. |
1682 |
// we have nothing to do here. Otherwise, make the bit ConstZero. |
| 1683 |
if ((*LHSBits)[i].isZero()) |
1683 |
if ((*LHSBits)[i].isZero()) |
| 1684 |
Bits[i] = (*LHSBits)[i]; |
1684 |
Bits[i] = (*LHSBits)[i]; |
| 1685 |
else |
1685 |
else |
| 1686 |
Bits[i] = ValueBit(ValueBit::ConstZero); |
1686 |
Bits[i] = ValueBit(ValueBit::ConstZero); |
| 1687 |
} |
1687 |
} |
| 1688 |
|
1688 |
|
| 1689 |
return std::make_pair(Interesting, &Bits); |
1689 |
return std::make_pair(Interesting, &Bits); |
| 1690 |
} |
1690 |
} |
| 1691 |
break; |
1691 |
break; |
| 1692 |
case ISD::OR: { |
1692 |
case ISD::OR: { |
| 1693 |
const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; |
1693 |
const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; |
| 1694 |
const auto &RHSBits = *getValueBits(V.getOperand(1), NumBits).second; |
1694 |
const auto &RHSBits = *getValueBits(V.getOperand(1), NumBits).second; |
| 1695 |
|
1695 |
|
| 1696 |
bool AllDisjoint = true; |
1696 |
bool AllDisjoint = true; |
| 1697 |
SDValue LastVal = SDValue(); |
1697 |
SDValue LastVal = SDValue(); |
| 1698 |
unsigned LastIdx = 0; |
1698 |
unsigned LastIdx = 0; |
| 1699 |
for (unsigned i = 0; i < NumBits; ++i) { |
1699 |
for (unsigned i = 0; i < NumBits; ++i) { |
| 1700 |
if (LHSBits[i].isZero() && RHSBits[i].isZero()) { |
1700 |
if (LHSBits[i].isZero() && RHSBits[i].isZero()) { |
| 1701 |
// If both inputs are known to be zero and one is ConstZero and |
1701 |
// If both inputs are known to be zero and one is ConstZero and |
| 1702 |
// another is VariableKnownToBeZero, we can select whichever |
1702 |
// another is VariableKnownToBeZero, we can select whichever |
| 1703 |
// we like. To minimize the number of bit groups, we select |
1703 |
// we like. To minimize the number of bit groups, we select |
| 1704 |
// VariableKnownToBeZero if this bit is the next bit of the same |
1704 |
// VariableKnownToBeZero if this bit is the next bit of the same |
| 1705 |
// input variable from the previous bit. Otherwise, we select |
1705 |
// input variable from the previous bit. Otherwise, we select |
| 1706 |
// ConstZero. |
1706 |
// ConstZero. |
| 1707 |
if (LHSBits[i].hasValue() && LHSBits[i].getValue() == LastVal && |
1707 |
if (LHSBits[i].hasValue() && LHSBits[i].getValue() == LastVal && |
| 1708 |
LHSBits[i].getValueBitIndex() == LastIdx + 1) |
1708 |
LHSBits[i].getValueBitIndex() == LastIdx + 1) |
| 1709 |
Bits[i] = LHSBits[i]; |
1709 |
Bits[i] = LHSBits[i]; |
| 1710 |
else if (RHSBits[i].hasValue() && RHSBits[i].getValue() == LastVal && |
1710 |
else if (RHSBits[i].hasValue() && RHSBits[i].getValue() == LastVal && |
| 1711 |
RHSBits[i].getValueBitIndex() == LastIdx + 1) |
1711 |
RHSBits[i].getValueBitIndex() == LastIdx + 1) |
| 1712 |
Bits[i] = RHSBits[i]; |
1712 |
Bits[i] = RHSBits[i]; |
| 1713 |
else |
1713 |
else |
| 1714 |
Bits[i] = ValueBit(ValueBit::ConstZero); |
1714 |
Bits[i] = ValueBit(ValueBit::ConstZero); |
| 1715 |
} |
1715 |
} |
| 1716 |
else if (LHSBits[i].isZero()) |
1716 |
else if (LHSBits[i].isZero()) |
| 1717 |
Bits[i] = RHSBits[i]; |
1717 |
Bits[i] = RHSBits[i]; |
| 1718 |
else if (RHSBits[i].isZero()) |
1718 |
else if (RHSBits[i].isZero()) |
| 1719 |
Bits[i] = LHSBits[i]; |
1719 |
Bits[i] = LHSBits[i]; |
| 1720 |
else { |
1720 |
else { |
| 1721 |
AllDisjoint = false; |
1721 |
AllDisjoint = false; |
| 1722 |
break; |
1722 |
break; |
| 1723 |
} |
1723 |
} |
| 1724 |
// We remember the value and bit index of this bit. |
1724 |
// We remember the value and bit index of this bit. |
| 1725 |
if (Bits[i].hasValue()) { |
1725 |
if (Bits[i].hasValue()) { |
| 1726 |
LastVal = Bits[i].getValue(); |
1726 |
LastVal = Bits[i].getValue(); |
| 1727 |
LastIdx = Bits[i].getValueBitIndex(); |
1727 |
LastIdx = Bits[i].getValueBitIndex(); |
| 1728 |
} |
1728 |
} |
| 1729 |
else { |
1729 |
else { |
| 1730 |
if (LastVal) LastVal = SDValue(); |
1730 |
if (LastVal) LastVal = SDValue(); |
| 1731 |
LastIdx = 0; |
1731 |
LastIdx = 0; |
| 1732 |
} |
1732 |
} |
| 1733 |
} |
1733 |
} |
| 1734 |
|
1734 |
|
| 1735 |
if (!AllDisjoint) |
1735 |
if (!AllDisjoint) |
| 1736 |
break; |
1736 |
break; |
| 1737 |
|
1737 |
|
| 1738 |
return std::make_pair(Interesting = true, &Bits); |
1738 |
return std::make_pair(Interesting = true, &Bits); |
| 1739 |
} |
1739 |
} |
| 1740 |
case ISD::ZERO_EXTEND: { |
1740 |
case ISD::ZERO_EXTEND: { |
| 1741 |
// We support only the case with zero extension from i32 to i64 so far. |
1741 |
// We support only the case with zero extension from i32 to i64 so far. |
| 1742 |
if (V.getValueType() != MVT::i64 || |
1742 |
if (V.getValueType() != MVT::i64 || |
| 1743 |
V.getOperand(0).getValueType() != MVT::i32) |
1743 |
V.getOperand(0).getValueType() != MVT::i32) |
| 1744 |
break; |
1744 |
break; |
| 1745 |
|
1745 |
|
| 1746 |
const SmallVector *LHSBits; |
1746 |
const SmallVector *LHSBits; |
| 1747 |
const unsigned NumOperandBits = 32; |
1747 |
const unsigned NumOperandBits = 32; |
| 1748 |
std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), |
1748 |
std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), |
| 1749 |
NumOperandBits); |
1749 |
NumOperandBits); |
| 1750 |
|
1750 |
|
| 1751 |
for (unsigned i = 0; i < NumOperandBits; ++i) |
1751 |
for (unsigned i = 0; i < NumOperandBits; ++i) |
| 1752 |
Bits[i] = (*LHSBits)[i]; |
1752 |
Bits[i] = (*LHSBits)[i]; |
| 1753 |
|
1753 |
|
| 1754 |
for (unsigned i = NumOperandBits; i < NumBits; ++i) |
1754 |
for (unsigned i = NumOperandBits; i < NumBits; ++i) |
| 1755 |
Bits[i] = ValueBit(ValueBit::ConstZero); |
1755 |
Bits[i] = ValueBit(ValueBit::ConstZero); |
| 1756 |
|
1756 |
|
| 1757 |
return std::make_pair(Interesting, &Bits); |
1757 |
return std::make_pair(Interesting, &Bits); |
| 1758 |
} |
1758 |
} |
| 1759 |
case ISD::TRUNCATE: { |
1759 |
case ISD::TRUNCATE: { |
| 1760 |
EVT FromType = V.getOperand(0).getValueType(); |
1760 |
EVT FromType = V.getOperand(0).getValueType(); |
| 1761 |
EVT ToType = V.getValueType(); |
1761 |
EVT ToType = V.getValueType(); |
| 1762 |
// We support only the case with truncate from i64 to i32. |
1762 |
// We support only the case with truncate from i64 to i32. |
| 1763 |
if (FromType != MVT::i64 || ToType != MVT::i32) |
1763 |
if (FromType != MVT::i64 || ToType != MVT::i32) |
| 1764 |
break; |
1764 |
break; |
| 1765 |
const unsigned NumAllBits = FromType.getSizeInBits(); |
1765 |
const unsigned NumAllBits = FromType.getSizeInBits(); |
| 1766 |
SmallVector *InBits; |
1766 |
SmallVector *InBits; |
| 1767 |
std::tie(Interesting, InBits) = getValueBits(V.getOperand(0), |
1767 |
std::tie(Interesting, InBits) = getValueBits(V.getOperand(0), |
| 1768 |
NumAllBits); |
1768 |
NumAllBits); |
| 1769 |
const unsigned NumValidBits = ToType.getSizeInBits(); |
1769 |
const unsigned NumValidBits = ToType.getSizeInBits(); |
| 1770 |
|
1770 |
|
| 1771 |
// A 32-bit instruction cannot touch upper 32-bit part of 64-bit value. |
1771 |
// A 32-bit instruction cannot touch upper 32-bit part of 64-bit value. |
| 1772 |
// So, we cannot include this truncate. |
1772 |
// So, we cannot include this truncate. |
| 1773 |
bool UseUpper32bit = false; |
1773 |
bool UseUpper32bit = false; |
| 1774 |
for (unsigned i = 0; i < NumValidBits; ++i) |
1774 |
for (unsigned i = 0; i < NumValidBits; ++i) |
| 1775 |
if ((*InBits)[i].hasValue() && (*InBits)[i].getValueBitIndex() >= 32) { |
1775 |
if ((*InBits)[i].hasValue() && (*InBits)[i].getValueBitIndex() >= 32) { |
| 1776 |
UseUpper32bit = true; |
1776 |
UseUpper32bit = true; |
| 1777 |
break; |
1777 |
break; |
| 1778 |
} |
1778 |
} |
| 1779 |
if (UseUpper32bit) |
1779 |
if (UseUpper32bit) |
| 1780 |
break; |
1780 |
break; |
| 1781 |
|
1781 |
|
| 1782 |
for (unsigned i = 0; i < NumValidBits; ++i) |
1782 |
for (unsigned i = 0; i < NumValidBits; ++i) |
| 1783 |
Bits[i] = (*InBits)[i]; |
1783 |
Bits[i] = (*InBits)[i]; |
| 1784 |
|
1784 |
|
| 1785 |
return std::make_pair(Interesting, &Bits); |
1785 |
return std::make_pair(Interesting, &Bits); |
| 1786 |
} |
1786 |
} |
| 1787 |
case ISD::AssertZext: { |
1787 |
case ISD::AssertZext: { |
| 1788 |
// For AssertZext, we look through the operand and |
1788 |
// For AssertZext, we look through the operand and |
| 1789 |
// mark the bits known to be zero. |
1789 |
// mark the bits known to be zero. |
| 1790 |
const SmallVector *LHSBits; |
1790 |
const SmallVector *LHSBits; |
| 1791 |
std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), |
1791 |
std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), |
| 1792 |
NumBits); |
1792 |
NumBits); |
| 1793 |
|
1793 |
|
| 1794 |
EVT FromType = cast(V.getOperand(1))->getVT(); |
1794 |
EVT FromType = cast(V.getOperand(1))->getVT(); |
| 1795 |
const unsigned NumValidBits = FromType.getSizeInBits(); |
1795 |
const unsigned NumValidBits = FromType.getSizeInBits(); |
| 1796 |
for (unsigned i = 0; i < NumValidBits; ++i) |
1796 |
for (unsigned i = 0; i < NumValidBits; ++i) |
| 1797 |
Bits[i] = (*LHSBits)[i]; |
1797 |
Bits[i] = (*LHSBits)[i]; |
| 1798 |
|
1798 |
|
| 1799 |
// These bits are known to be zero but the AssertZext may be from a value |
1799 |
// These bits are known to be zero but the AssertZext may be from a value |
| 1800 |
// that already has some constant zero bits (i.e. from a masking and). |
1800 |
// that already has some constant zero bits (i.e. from a masking and). |
| 1801 |
for (unsigned i = NumValidBits; i < NumBits; ++i) |
1801 |
for (unsigned i = NumValidBits; i < NumBits; ++i) |
| 1802 |
Bits[i] = (*LHSBits)[i].hasValue() |
1802 |
Bits[i] = (*LHSBits)[i].hasValue() |
| 1803 |
? ValueBit((*LHSBits)[i].getValue(), |
1803 |
? ValueBit((*LHSBits)[i].getValue(), |
| 1804 |
(*LHSBits)[i].getValueBitIndex(), |
1804 |
(*LHSBits)[i].getValueBitIndex(), |
| 1805 |
ValueBit::VariableKnownToBeZero) |
1805 |
ValueBit::VariableKnownToBeZero) |
| 1806 |
: ValueBit(ValueBit::ConstZero); |
1806 |
: ValueBit(ValueBit::ConstZero); |
| 1807 |
|
1807 |
|
| 1808 |
return std::make_pair(Interesting, &Bits); |
1808 |
return std::make_pair(Interesting, &Bits); |
| 1809 |
} |
1809 |
} |
| 1810 |
case ISD::LOAD: |
1810 |
case ISD::LOAD: |
| 1811 |
LoadSDNode *LD = cast(V); |
1811 |
LoadSDNode *LD = cast(V); |
| 1812 |
if (ISD::isZEXTLoad(V.getNode()) && V.getResNo() == 0) { |
1812 |
if (ISD::isZEXTLoad(V.getNode()) && V.getResNo() == 0) { |
| 1813 |
EVT VT = LD->getMemoryVT(); |
1813 |
EVT VT = LD->getMemoryVT(); |
| 1814 |
const unsigned NumValidBits = VT.getSizeInBits(); |
1814 |
const unsigned NumValidBits = VT.getSizeInBits(); |
| 1815 |
|
1815 |
|
| 1816 |
for (unsigned i = 0; i < NumValidBits; ++i) |
1816 |
for (unsigned i = 0; i < NumValidBits; ++i) |
| 1817 |
Bits[i] = ValueBit(V, i); |
1817 |
Bits[i] = ValueBit(V, i); |
| 1818 |
|
1818 |
|
| 1819 |
// These bits are known to be zero. |
1819 |
// These bits are known to be zero. |
| 1820 |
for (unsigned i = NumValidBits; i < NumBits; ++i) |
1820 |
for (unsigned i = NumValidBits; i < NumBits; ++i) |
| 1821 |
Bits[i] = ValueBit(V, i, ValueBit::VariableKnownToBeZero); |
1821 |
Bits[i] = ValueBit(V, i, ValueBit::VariableKnownToBeZero); |
| 1822 |
|
1822 |
|
| 1823 |
// Zero-extending load itself cannot be optimized. So, it is not |
1823 |
// Zero-extending load itself cannot be optimized. So, it is not |
| 1824 |
// interesting by itself though it gives useful information. |
1824 |
// interesting by itself though it gives useful information. |
| 1825 |
return std::make_pair(Interesting = false, &Bits); |
1825 |
return std::make_pair(Interesting = false, &Bits); |
| 1826 |
} |
1826 |
} |
| 1827 |
break; |
1827 |
break; |
| 1828 |
} |
1828 |
} |
| 1829 |
|
1829 |
|
| 1830 |
for (unsigned i = 0; i < NumBits; ++i) |
1830 |
for (unsigned i = 0; i < NumBits; ++i) |
| 1831 |
Bits[i] = ValueBit(V, i); |
1831 |
Bits[i] = ValueBit(V, i); |
| 1832 |
|
1832 |
|
| 1833 |
return std::make_pair(Interesting = false, &Bits); |
1833 |
return std::make_pair(Interesting = false, &Bits); |
| 1834 |
} |
1834 |
} |
| 1835 |
|
1835 |
|
| 1836 |
// For each value (except the constant ones), compute the left-rotate amount |
1836 |
// For each value (except the constant ones), compute the left-rotate amount |
| 1837 |
// to get it from its original to final position. |
1837 |
// to get it from its original to final position. |
| 1838 |
void computeRotationAmounts() { |
1838 |
void computeRotationAmounts() { |
| 1839 |
NeedMask = false; |
1839 |
NeedMask = false; |
| 1840 |
RLAmt.resize(Bits.size()); |
1840 |
RLAmt.resize(Bits.size()); |
| 1841 |
for (unsigned i = 0; i < Bits.size(); ++i) |
1841 |
for (unsigned i = 0; i < Bits.size(); ++i) |
| 1842 |
if (Bits[i].hasValue()) { |
1842 |
if (Bits[i].hasValue()) { |
| 1843 |
unsigned VBI = Bits[i].getValueBitIndex(); |
1843 |
unsigned VBI = Bits[i].getValueBitIndex(); |
| 1844 |
if (i >= VBI) |
1844 |
if (i >= VBI) |
| 1845 |
RLAmt[i] = i - VBI; |
1845 |
RLAmt[i] = i - VBI; |
| 1846 |
else |
1846 |
else |
| 1847 |
RLAmt[i] = Bits.size() - (VBI - i); |
1847 |
RLAmt[i] = Bits.size() - (VBI - i); |
| 1848 |
} else if (Bits[i].isZero()) { |
1848 |
} else if (Bits[i].isZero()) { |
| 1849 |
NeedMask = true; |
1849 |
NeedMask = true; |
| 1850 |
RLAmt[i] = UINT32_MAX; |
1850 |
RLAmt[i] = UINT32_MAX; |
| 1851 |
} else { |
1851 |
} else { |
| 1852 |
llvm_unreachable("Unknown value bit type"); |
1852 |
llvm_unreachable("Unknown value bit type"); |
| 1853 |
} |
1853 |
} |
| 1854 |
} |
1854 |
} |
| 1855 |
|
1855 |
|
| 1856 |
// Collect groups of consecutive bits with the same underlying value and |
1856 |
// Collect groups of consecutive bits with the same underlying value and |
| 1857 |
// rotation factor. If we're doing late masking, we ignore zeros, otherwise |
1857 |
// rotation factor. If we're doing late masking, we ignore zeros, otherwise |
| 1858 |
// they break up groups. |
1858 |
// they break up groups. |
| 1859 |
void collectBitGroups(bool LateMask) { |
1859 |
void collectBitGroups(bool LateMask) { |
| 1860 |
BitGroups.clear(); |
1860 |
BitGroups.clear(); |
| 1861 |
|
1861 |
|
| 1862 |
unsigned LastRLAmt = RLAmt[0]; |
1862 |
unsigned LastRLAmt = RLAmt[0]; |
| 1863 |
SDValue LastValue = Bits[0].hasValue() ? Bits[0].getValue() : SDValue(); |
1863 |
SDValue LastValue = Bits[0].hasValue() ? Bits[0].getValue() : SDValue(); |
| 1864 |
unsigned LastGroupStartIdx = 0; |
1864 |
unsigned LastGroupStartIdx = 0; |
| 1865 |
bool IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue(); |
1865 |
bool IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue(); |
| 1866 |
for (unsigned i = 1; i < Bits.size(); ++i) { |
1866 |
for (unsigned i = 1; i < Bits.size(); ++i) { |
| 1867 |
unsigned ThisRLAmt = RLAmt[i]; |
1867 |
unsigned ThisRLAmt = RLAmt[i]; |
| 1868 |
SDValue ThisValue = Bits[i].hasValue() ? Bits[i].getValue() : SDValue(); |
1868 |
SDValue ThisValue = Bits[i].hasValue() ? Bits[i].getValue() : SDValue(); |
| 1869 |
if (LateMask && !ThisValue) { |
1869 |
if (LateMask && !ThisValue) { |
| 1870 |
ThisValue = LastValue; |
1870 |
ThisValue = LastValue; |
| 1871 |
ThisRLAmt = LastRLAmt; |
1871 |
ThisRLAmt = LastRLAmt; |
| 1872 |
// If we're doing late masking, then the first bit group always starts |
1872 |
// If we're doing late masking, then the first bit group always starts |
| 1873 |
// at zero (even if the first bits were zero). |
1873 |
// at zero (even if the first bits were zero). |
| 1874 |
if (BitGroups.empty()) |
1874 |
if (BitGroups.empty()) |
| 1875 |
LastGroupStartIdx = 0; |
1875 |
LastGroupStartIdx = 0; |
| 1876 |
} |
1876 |
} |
| 1877 |
|
1877 |
|
| 1878 |
// If this bit is known to be zero and the current group is a bit group |
1878 |
// If this bit is known to be zero and the current group is a bit group |
| 1879 |
// of zeros, we do not need to terminate the current bit group even the |
1879 |
// of zeros, we do not need to terminate the current bit group even the |
| 1880 |
// Value or RLAmt does not match here. Instead, we terminate this group |
1880 |
// Value or RLAmt does not match here. Instead, we terminate this group |
| 1881 |
// when the first non-zero bit appears later. |
1881 |
// when the first non-zero bit appears later. |
| 1882 |
if (IsGroupOfZeros && Bits[i].isZero()) |
1882 |
if (IsGroupOfZeros && Bits[i].isZero()) |
| 1883 |
continue; |
1883 |
continue; |
| 1884 |
|
1884 |
|
| 1885 |
// If this bit has the same underlying value and the same rotate factor as |
1885 |
// If this bit has the same underlying value and the same rotate factor as |
| 1886 |
// the last one, then they're part of the same group. |
1886 |
// the last one, then they're part of the same group. |
| 1887 |
if (ThisRLAmt == LastRLAmt && ThisValue == LastValue) |
1887 |
if (ThisRLAmt == LastRLAmt && ThisValue == LastValue) |
| 1888 |
// We cannot continue the current group if this bits is not known to |
1888 |
// We cannot continue the current group if this bits is not known to |
| 1889 |
// be zero in a bit group of zeros. |
1889 |
// be zero in a bit group of zeros. |
| 1890 |
if (!(IsGroupOfZeros && ThisValue && !Bits[i].isZero())) |
1890 |
if (!(IsGroupOfZeros && ThisValue && !Bits[i].isZero())) |
| 1891 |
continue; |
1891 |
continue; |
| 1892 |
|
1892 |
|
| 1893 |
if (LastValue.getNode()) |
1893 |
if (LastValue.getNode()) |
| 1894 |
BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx, |
1894 |
BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx, |
| 1895 |
i-1)); |
1895 |
i-1)); |
| 1896 |
LastRLAmt = ThisRLAmt; |
1896 |
LastRLAmt = ThisRLAmt; |
| 1897 |
LastValue = ThisValue; |
1897 |
LastValue = ThisValue; |
| 1898 |
LastGroupStartIdx = i; |
1898 |
LastGroupStartIdx = i; |
| 1899 |
IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue(); |
1899 |
IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue(); |
| 1900 |
} |
1900 |
} |
| 1901 |
if (LastValue.getNode()) |
1901 |
if (LastValue.getNode()) |
| 1902 |
BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx, |
1902 |
BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx, |
| 1903 |
Bits.size()-1)); |
1903 |
Bits.size()-1)); |
| 1904 |
|
1904 |
|
| 1905 |
if (BitGroups.empty()) |
1905 |
if (BitGroups.empty()) |
| 1906 |
return; |
1906 |
return; |
| 1907 |
|
1907 |
|
| 1908 |
// We might be able to combine the first and last groups. |
1908 |
// We might be able to combine the first and last groups. |
| 1909 |
if (BitGroups.size() > 1) { |
1909 |
if (BitGroups.size() > 1) { |
| 1910 |
// If the first and last groups are the same, then remove the first group |
1910 |
// If the first and last groups are the same, then remove the first group |
| 1911 |
// in favor of the last group, making the ending index of the last group |
1911 |
// in favor of the last group, making the ending index of the last group |
| 1912 |
// equal to the ending index of the to-be-removed first group. |
1912 |
// equal to the ending index of the to-be-removed first group. |
| 1913 |
if (BitGroups[0].StartIdx == 0 && |
1913 |
if (BitGroups[0].StartIdx == 0 && |
| 1914 |
BitGroups[BitGroups.size()-1].EndIdx == Bits.size()-1 && |
1914 |
BitGroups[BitGroups.size()-1].EndIdx == Bits.size()-1 && |
| 1915 |
BitGroups[0].V == BitGroups[BitGroups.size()-1].V && |
1915 |
BitGroups[0].V == BitGroups[BitGroups.size()-1].V && |
| 1916 |
BitGroups[0].RLAmt == BitGroups[BitGroups.size()-1].RLAmt) { |
1916 |
BitGroups[0].RLAmt == BitGroups[BitGroups.size()-1].RLAmt) { |
| 1917 |
LLVM_DEBUG(dbgs() << "\tcombining final bit group with initial one\n"); |
1917 |
LLVM_DEBUG(dbgs() << "\tcombining final bit group with initial one\n"); |
| 1918 |
BitGroups[BitGroups.size()-1].EndIdx = BitGroups[0].EndIdx; |
1918 |
BitGroups[BitGroups.size()-1].EndIdx = BitGroups[0].EndIdx; |
| 1919 |
BitGroups.erase(BitGroups.begin()); |
1919 |
BitGroups.erase(BitGroups.begin()); |
| 1920 |
} |
1920 |
} |
| 1921 |
} |
1921 |
} |
| 1922 |
} |
1922 |
} |
| 1923 |
|
1923 |
|
| 1924 |
// Take all (SDValue, RLAmt) pairs and sort them by the number of groups |
1924 |
// Take all (SDValue, RLAmt) pairs and sort them by the number of groups |
| 1925 |
// associated with each. If the number of groups are same, we prefer a group |
1925 |
// associated with each. If the number of groups are same, we prefer a group |
| 1926 |
// which does not require rotate, i.e. RLAmt is 0, to avoid the first rotate |
1926 |
// which does not require rotate, i.e. RLAmt is 0, to avoid the first rotate |
| 1927 |
// instruction. If there is a degeneracy, pick the one that occurs |
1927 |
// instruction. If there is a degeneracy, pick the one that occurs |
| 1928 |
// first (in the final value). |
1928 |
// first (in the final value). |
| 1929 |
void collectValueRotInfo() { |
1929 |
void collectValueRotInfo() { |
| 1930 |
ValueRots.clear(); |
1930 |
ValueRots.clear(); |
| 1931 |
|
1931 |
|
| 1932 |
for (auto &BG : BitGroups) { |
1932 |
for (auto &BG : BitGroups) { |
| 1933 |
unsigned RLAmtKey = BG.RLAmt + (BG.Repl32 ? 64 : 0); |
1933 |
unsigned RLAmtKey = BG.RLAmt + (BG.Repl32 ? 64 : 0); |
| 1934 |
ValueRotInfo &VRI = ValueRots[std::make_pair(BG.V, RLAmtKey)]; |
1934 |
ValueRotInfo &VRI = ValueRots[std::make_pair(BG.V, RLAmtKey)]; |
| 1935 |
VRI.V = BG.V; |
1935 |
VRI.V = BG.V; |
| 1936 |
VRI.RLAmt = BG.RLAmt; |
1936 |
VRI.RLAmt = BG.RLAmt; |
| 1937 |
VRI.Repl32 = BG.Repl32; |
1937 |
VRI.Repl32 = BG.Repl32; |
| 1938 |
VRI.NumGroups += 1; |
1938 |
VRI.NumGroups += 1; |
| 1939 |
VRI.FirstGroupStartIdx = std::min(VRI.FirstGroupStartIdx, BG.StartIdx); |
1939 |
VRI.FirstGroupStartIdx = std::min(VRI.FirstGroupStartIdx, BG.StartIdx); |
| 1940 |
} |
1940 |
} |
| 1941 |
|
1941 |
|
| 1942 |
// Now that we've collected the various ValueRotInfo instances, we need to |
1942 |
// Now that we've collected the various ValueRotInfo instances, we need to |
| 1943 |
// sort them. |
1943 |
// sort them. |
| 1944 |
ValueRotsVec.clear(); |
1944 |
ValueRotsVec.clear(); |
| 1945 |
for (auto &I : ValueRots) { |
1945 |
for (auto &I : ValueRots) { |
| 1946 |
ValueRotsVec.push_back(I.second); |
1946 |
ValueRotsVec.push_back(I.second); |
| 1947 |
} |
1947 |
} |
| 1948 |
llvm::sort(ValueRotsVec); |
1948 |
llvm::sort(ValueRotsVec); |
| 1949 |
} |
1949 |
} |
| 1950 |
|
1950 |
|
| 1951 |
// In 64-bit mode, rlwinm and friends have a rotation operator that |
1951 |
// In 64-bit mode, rlwinm and friends have a rotation operator that |
| 1952 |
// replicates the low-order 32 bits into the high-order 32-bits. The mask |
1952 |
// replicates the low-order 32 bits into the high-order 32-bits. The mask |
| 1953 |
// indices of these instructions can only be in the lower 32 bits, so they |
1953 |
// indices of these instructions can only be in the lower 32 bits, so they |
| 1954 |
// can only represent some 64-bit bit groups. However, when they can be used, |
1954 |
// can only represent some 64-bit bit groups. However, when they can be used, |
| 1955 |
// the 32-bit replication can be used to represent, as a single bit group, |
1955 |
// the 32-bit replication can be used to represent, as a single bit group, |
| 1956 |
// otherwise separate bit groups. We'll convert to replicated-32-bit bit |
1956 |
// otherwise separate bit groups. We'll convert to replicated-32-bit bit |
| 1957 |
// groups when possible. Returns true if any of the bit groups were |
1957 |
// groups when possible. Returns true if any of the bit groups were |
| 1958 |
// converted. |
1958 |
// converted. |
| 1959 |
void assignRepl32BitGroups() { |
1959 |
void assignRepl32BitGroups() { |
| 1960 |
// If we have bits like this: |
1960 |
// If we have bits like this: |
| 1961 |
// |
1961 |
// |
| 1962 |
// Indices: 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 |
1962 |
// Indices: 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 |
| 1963 |
// V bits: ... 7 6 5 4 3 2 1 0 31 30 29 28 27 26 25 24 |
1963 |
// V bits: ... 7 6 5 4 3 2 1 0 31 30 29 28 27 26 25 24 |
| 1964 |
// Groups: | RLAmt = 8 | RLAmt = 40 | |
1964 |
// Groups: | RLAmt = 8 | RLAmt = 40 | |
| 1965 |
// |
1965 |
// |
| 1966 |
// But, making use of a 32-bit operation that replicates the low-order 32 |
1966 |
// But, making use of a 32-bit operation that replicates the low-order 32 |
| 1967 |
// bits into the high-order 32 bits, this can be one bit group with a RLAmt |
1967 |
// bits into the high-order 32 bits, this can be one bit group with a RLAmt |
| 1968 |
// of 8. |
1968 |
// of 8. |
| 1969 |
|
1969 |
|
| 1970 |
auto IsAllLow32 = [this](BitGroup & BG) { |
1970 |
auto IsAllLow32 = [this](BitGroup & BG) { |
| 1971 |
if (BG.StartIdx <= BG.EndIdx) { |
1971 |
if (BG.StartIdx <= BG.EndIdx) { |
| 1972 |
for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) { |
1972 |
for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) { |
| 1973 |
if (!Bits[i].hasValue()) |
1973 |
if (!Bits[i].hasValue()) |
| 1974 |
continue; |
1974 |
continue; |
| 1975 |
if (Bits[i].getValueBitIndex() >= 32) |
1975 |
if (Bits[i].getValueBitIndex() >= 32) |
| 1976 |
return false; |
1976 |
return false; |
| 1977 |
} |
1977 |
} |
| 1978 |
} else { |
1978 |
} else { |
| 1979 |
for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) { |
1979 |
for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) { |
| 1980 |
if (!Bits[i].hasValue()) |
1980 |
if (!Bits[i].hasValue()) |
| 1981 |
continue; |
1981 |
continue; |
| 1982 |
if (Bits[i].getValueBitIndex() >= 32) |
1982 |
if (Bits[i].getValueBitIndex() >= 32) |
| 1983 |
return false; |
1983 |
return false; |
| 1984 |
} |
1984 |
} |
| 1985 |
for (unsigned i = 0; i <= BG.EndIdx; ++i) { |
1985 |
for (unsigned i = 0; i <= BG.EndIdx; ++i) { |
| 1986 |
if (!Bits[i].hasValue()) |
1986 |
if (!Bits[i].hasValue()) |
| 1987 |
continue; |
1987 |
continue; |
| 1988 |
if (Bits[i].getValueBitIndex() >= 32) |
1988 |
if (Bits[i].getValueBitIndex() >= 32) |
| 1989 |
return false; |
1989 |
return false; |
| 1990 |
} |
1990 |
} |
| 1991 |
} |
1991 |
} |
| 1992 |
|
1992 |
|
| 1993 |
return true; |
1993 |
return true; |
| 1994 |
}; |
1994 |
}; |
| 1995 |
|
1995 |
|
| 1996 |
for (auto &BG : BitGroups) { |
1996 |
for (auto &BG : BitGroups) { |
| 1997 |
// If this bit group has RLAmt of 0 and will not be merged with |
1997 |
// If this bit group has RLAmt of 0 and will not be merged with |
| 1998 |
// another bit group, we don't benefit from Repl32. We don't mark |
1998 |
// another bit group, we don't benefit from Repl32. We don't mark |
| 1999 |
// such group to give more freedom for later instruction selection. |
1999 |
// such group to give more freedom for later instruction selection. |
| 2000 |
if (BG.RLAmt == 0) { |
2000 |
if (BG.RLAmt == 0) { |
| 2001 |
auto PotentiallyMerged = [this](BitGroup & BG) { |
2001 |
auto PotentiallyMerged = [this](BitGroup & BG) { |
| 2002 |
for (auto &BG2 : BitGroups) |
2002 |
for (auto &BG2 : BitGroups) |
| 2003 |
if (&BG != &BG2 && BG.V == BG2.V && |
2003 |
if (&BG != &BG2 && BG.V == BG2.V && |
| 2004 |
(BG2.RLAmt == 0 || BG2.RLAmt == 32)) |
2004 |
(BG2.RLAmt == 0 || BG2.RLAmt == 32)) |
| 2005 |
return true; |
2005 |
return true; |
| 2006 |
return false; |
2006 |
return false; |
| 2007 |
}; |
2007 |
}; |
| 2008 |
if (!PotentiallyMerged(BG)) |
2008 |
if (!PotentiallyMerged(BG)) |
| 2009 |
continue; |
2009 |
continue; |
| 2010 |
} |
2010 |
} |
| 2011 |
if (BG.StartIdx < 32 && BG.EndIdx < 32) { |
2011 |
if (BG.StartIdx < 32 && BG.EndIdx < 32) { |
| 2012 |
if (IsAllLow32(BG)) { |
2012 |
if (IsAllLow32(BG)) { |
| 2013 |
if (BG.RLAmt >= 32) { |
2013 |
if (BG.RLAmt >= 32) { |
| 2014 |
BG.RLAmt -= 32; |
2014 |
BG.RLAmt -= 32; |
| 2015 |
BG.Repl32CR = true; |
2015 |
BG.Repl32CR = true; |
| 2016 |
} |
2016 |
} |
| 2017 |
|
2017 |
|
| 2018 |
BG.Repl32 = true; |
2018 |
BG.Repl32 = true; |
| 2019 |
|
2019 |
|
| 2020 |
LLVM_DEBUG(dbgs() << "\t32-bit replicated bit group for " |
2020 |
LLVM_DEBUG(dbgs() << "\t32-bit replicated bit group for " |
| 2021 |
<< BG.V.getNode() << " RLAmt = " << BG.RLAmt << " [" |
2021 |
<< BG.V.getNode() << " RLAmt = " << BG.RLAmt << " [" |
| 2022 |
<< BG.StartIdx << ", " << BG.EndIdx << "]\n"); |
2022 |
<< BG.StartIdx << ", " << BG.EndIdx << "]\n"); |
| 2023 |
} |
2023 |
} |
| 2024 |
} |
2024 |
} |
| 2025 |
} |
2025 |
} |
| 2026 |
|
2026 |
|
| 2027 |
// Now walk through the bit groups, consolidating where possible. |
2027 |
// Now walk through the bit groups, consolidating where possible. |
| 2028 |
for (auto I = BitGroups.begin(); I != BitGroups.end();) { |
2028 |
for (auto I = BitGroups.begin(); I != BitGroups.end();) { |
| 2029 |
// We might want to remove this bit group by merging it with the previous |
2029 |
// We might want to remove this bit group by merging it with the previous |
| 2030 |
// group (which might be the ending group). |
2030 |
// group (which might be the ending group). |
| 2031 |
auto IP = (I == BitGroups.begin()) ? |
2031 |
auto IP = (I == BitGroups.begin()) ? |
| 2032 |
std::prev(BitGroups.end()) : std::prev(I); |
2032 |
std::prev(BitGroups.end()) : std::prev(I); |
| 2033 |
if (I->Repl32 && IP->Repl32 && I->V == IP->V && I->RLAmt == IP->RLAmt && |
2033 |
if (I->Repl32 && IP->Repl32 && I->V == IP->V && I->RLAmt == IP->RLAmt && |
| 2034 |
I->StartIdx == (IP->EndIdx + 1) % 64 && I != IP) { |
2034 |
I->StartIdx == (IP->EndIdx + 1) % 64 && I != IP) { |
| 2035 |
|
2035 |
|
| 2036 |
LLVM_DEBUG(dbgs() << "\tcombining 32-bit replicated bit group for " |
2036 |
LLVM_DEBUG(dbgs() << "\tcombining 32-bit replicated bit group for " |
| 2037 |
<< I->V.getNode() << " RLAmt = " << I->RLAmt << " [" |
2037 |
<< I->V.getNode() << " RLAmt = " << I->RLAmt << " [" |
| 2038 |
<< I->StartIdx << ", " << I->EndIdx |
2038 |
<< I->StartIdx << ", " << I->EndIdx |
| 2039 |
<< "] with group with range [" << IP->StartIdx << ", " |
2039 |
<< "] with group with range [" << IP->StartIdx << ", " |
| 2040 |
<< IP->EndIdx << "]\n"); |
2040 |
<< IP->EndIdx << "]\n"); |
| 2041 |
|
2041 |
|
| 2042 |
IP->EndIdx = I->EndIdx; |
2042 |
IP->EndIdx = I->EndIdx; |
| 2043 |
IP->Repl32CR = IP->Repl32CR || I->Repl32CR; |
2043 |
IP->Repl32CR = IP->Repl32CR || I->Repl32CR; |
| 2044 |
IP->Repl32Coalesced = true; |
2044 |
IP->Repl32Coalesced = true; |
| 2045 |
I = BitGroups.erase(I); |
2045 |
I = BitGroups.erase(I); |
| 2046 |
continue; |
2046 |
continue; |
| 2047 |
} else { |
2047 |
} else { |
| 2048 |
// There is a special case worth handling: If there is a single group |
2048 |
// There is a special case worth handling: If there is a single group |
| 2049 |
// covering the entire upper 32 bits, and it can be merged with both |
2049 |
// covering the entire upper 32 bits, and it can be merged with both |
| 2050 |
// the next and previous groups (which might be the same group), then |
2050 |
// the next and previous groups (which might be the same group), then |
| 2051 |
// do so. If it is the same group (so there will be only one group in |
2051 |
// do so. If it is the same group (so there will be only one group in |
| 2052 |
// total), then we need to reverse the order of the range so that it |
2052 |
// total), then we need to reverse the order of the range so that it |
| 2053 |
// covers the entire 64 bits. |
2053 |
// covers the entire 64 bits. |
| 2054 |
if (I->StartIdx == 32 && I->EndIdx == 63) { |
2054 |
if (I->StartIdx == 32 && I->EndIdx == 63) { |
| 2055 |
assert(std::next(I) == BitGroups.end() && |
2055 |
assert(std::next(I) == BitGroups.end() && |
| 2056 |
"bit group ends at index 63 but there is another?"); |
2056 |
"bit group ends at index 63 but there is another?"); |
| 2057 |
auto IN = BitGroups.begin(); |
2057 |
auto IN = BitGroups.begin(); |
| 2058 |
|
2058 |
|
| 2059 |
if (IP->Repl32 && IN->Repl32 && I->V == IP->V && I->V == IN->V && |
2059 |
if (IP->Repl32 && IN->Repl32 && I->V == IP->V && I->V == IN->V && |
| 2060 |
(I->RLAmt % 32) == IP->RLAmt && (I->RLAmt % 32) == IN->RLAmt && |
2060 |
(I->RLAmt % 32) == IP->RLAmt && (I->RLAmt % 32) == IN->RLAmt && |
| 2061 |
IP->EndIdx == 31 && IN->StartIdx == 0 && I != IP && |
2061 |
IP->EndIdx == 31 && IN->StartIdx == 0 && I != IP && |
| 2062 |
IsAllLow32(*I)) { |
2062 |
IsAllLow32(*I)) { |
| 2063 |
|
2063 |
|
| 2064 |
LLVM_DEBUG(dbgs() << "\tcombining bit group for " << I->V.getNode() |
2064 |
LLVM_DEBUG(dbgs() << "\tcombining bit group for " << I->V.getNode() |
| 2065 |
<< " RLAmt = " << I->RLAmt << " [" << I->StartIdx |
2065 |
<< " RLAmt = " << I->RLAmt << " [" << I->StartIdx |
| 2066 |
<< ", " << I->EndIdx |
2066 |
<< ", " << I->EndIdx |
| 2067 |
<< "] with 32-bit replicated groups with ranges [" |
2067 |
<< "] with 32-bit replicated groups with ranges [" |
| 2068 |
<< IP->StartIdx << ", " << IP->EndIdx << "] and [" |
2068 |
<< IP->StartIdx << ", " << IP->EndIdx << "] and [" |
| 2069 |
<< IN->StartIdx << ", " << IN->EndIdx << "]\n"); |
2069 |
<< IN->StartIdx << ", " << IN->EndIdx << "]\n"); |
| 2070 |
|
2070 |
|
| 2071 |
if (IP == IN) { |
2071 |
if (IP == IN) { |
| 2072 |
// There is only one other group; change it to cover the whole |
2072 |
// There is only one other group; change it to cover the whole |
| 2073 |
// range (backward, so that it can still be Repl32 but cover the |
2073 |
// range (backward, so that it can still be Repl32 but cover the |
| 2074 |
// whole 64-bit range). |
2074 |
// whole 64-bit range). |
| 2075 |
IP->StartIdx = 31; |
2075 |
IP->StartIdx = 31; |
| 2076 |
IP->EndIdx = 30; |
2076 |
IP->EndIdx = 30; |
| 2077 |
IP->Repl32CR = IP->Repl32CR || I->RLAmt >= 32; |
2077 |
IP->Repl32CR = IP->Repl32CR || I->RLAmt >= 32; |
| 2078 |
IP->Repl32Coalesced = true; |
2078 |
IP->Repl32Coalesced = true; |
| 2079 |
I = BitGroups.erase(I); |
2079 |
I = BitGroups.erase(I); |
| 2080 |
} else { |
2080 |
} else { |
| 2081 |
// There are two separate groups, one before this group and one |
2081 |
// There are two separate groups, one before this group and one |
| 2082 |
// after us (at the beginning). We're going to remove this group, |
2082 |
// after us (at the beginning). We're going to remove this group, |
| 2083 |
// but also the group at the very beginning. |
2083 |
// but also the group at the very beginning. |
| 2084 |
IP->EndIdx = IN->EndIdx; |
2084 |
IP->EndIdx = IN->EndIdx; |
| 2085 |
IP->Repl32CR = IP->Repl32CR || IN->Repl32CR || I->RLAmt >= 32; |
2085 |
IP->Repl32CR = IP->Repl32CR || IN->Repl32CR || I->RLAmt >= 32; |
| 2086 |
IP->Repl32Coalesced = true; |
2086 |
IP->Repl32Coalesced = true; |
| 2087 |
I = BitGroups.erase(I); |
2087 |
I = BitGroups.erase(I); |
| 2088 |
BitGroups.erase(BitGroups.begin()); |
2088 |
BitGroups.erase(BitGroups.begin()); |
| 2089 |
} |
2089 |
} |
| 2090 |
|
2090 |
|
| 2091 |
// This must be the last group in the vector (and we might have |
2091 |
// This must be the last group in the vector (and we might have |
| 2092 |
// just invalidated the iterator above), so break here. |
2092 |
// just invalidated the iterator above), so break here. |
| 2093 |
break; |
2093 |
break; |
| 2094 |
} |
2094 |
} |
| 2095 |
} |
2095 |
} |
| 2096 |
} |
2096 |
} |
| 2097 |
|
2097 |
|
| 2098 |
++I; |
2098 |
++I; |
| 2099 |
} |
2099 |
} |
| 2100 |
} |
2100 |
} |
| 2101 |
|
2101 |
|
| 2102 |
SDValue getI32Imm(unsigned Imm, const SDLoc &dl) { |
2102 |
SDValue getI32Imm(unsigned Imm, const SDLoc &dl) { |
| 2103 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
2103 |
return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
| 2104 |
} |
2104 |
} |
| 2105 |
|
2105 |
|
| 2106 |
uint64_t getZerosMask() { |
2106 |
uint64_t getZerosMask() { |
| 2107 |
uint64_t Mask = 0; |
2107 |
uint64_t Mask = 0; |
| 2108 |
for (unsigned i = 0; i < Bits.size(); ++i) { |
2108 |
for (unsigned i = 0; i < Bits.size(); ++i) { |
| 2109 |
if (Bits[i].hasValue()) |
2109 |
if (Bits[i].hasValue()) |
| 2110 |
continue; |
2110 |
continue; |
| 2111 |
Mask |= (UINT64_C(1) << i); |
2111 |
Mask |= (UINT64_C(1) << i); |
| 2112 |
} |
2112 |
} |
| 2113 |
|
2113 |
|
| 2114 |
return ~Mask; |
2114 |
return ~Mask; |
| 2115 |
} |
2115 |
} |
| 2116 |
|
2116 |
|
| 2117 |
// This method extends an input value to 64 bit if input is 32-bit integer. |
2117 |
// This method extends an input value to 64 bit if input is 32-bit integer. |
| 2118 |
// While selecting instructions in BitPermutationSelector in 64-bit mode, |
2118 |
// While selecting instructions in BitPermutationSelector in 64-bit mode, |
| 2119 |
// an input value can be a 32-bit integer if a ZERO_EXTEND node is included. |
2119 |
// an input value can be a 32-bit integer if a ZERO_EXTEND node is included. |
| 2120 |
// In such case, we extend it to 64 bit to be consistent with other values. |
2120 |
// In such case, we extend it to 64 bit to be consistent with other values. |
| 2121 |
SDValue ExtendToInt64(SDValue V, const SDLoc &dl) { |
2121 |
SDValue ExtendToInt64(SDValue V, const SDLoc &dl) { |
| 2122 |
if (V.getValueSizeInBits() == 64) |
2122 |
if (V.getValueSizeInBits() == 64) |
| 2123 |
return V; |
2123 |
return V; |
| 2124 |
|
2124 |
|
| 2125 |
assert(V.getValueSizeInBits() == 32); |
2125 |
assert(V.getValueSizeInBits() == 32); |
| 2126 |
SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); |
2126 |
SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); |
| 2127 |
SDValue ImDef = SDValue(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, |
2127 |
SDValue ImDef = SDValue(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, |
| 2128 |
MVT::i64), 0); |
2128 |
MVT::i64), 0); |
| 2129 |
SDValue ExtVal = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, |
2129 |
SDValue ExtVal = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, |
| 2130 |
MVT::i64, ImDef, V, |
2130 |
MVT::i64, ImDef, V, |
| 2131 |
SubRegIdx), 0); |
2131 |
SubRegIdx), 0); |
| 2132 |
return ExtVal; |
2132 |
return ExtVal; |
| 2133 |
} |
2133 |
} |
| 2134 |
|
2134 |
|
| 2135 |
SDValue TruncateToInt32(SDValue V, const SDLoc &dl) { |
2135 |
SDValue TruncateToInt32(SDValue V, const SDLoc &dl) { |
| 2136 |
if (V.getValueSizeInBits() == 32) |
2136 |
if (V.getValueSizeInBits() == 32) |
| 2137 |
return V; |
2137 |
return V; |
| 2138 |
|
2138 |
|
| 2139 |
assert(V.getValueSizeInBits() == 64); |
2139 |
assert(V.getValueSizeInBits() == 64); |
| 2140 |
SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); |
2140 |
SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); |
| 2141 |
SDValue SubVal = SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, |
2141 |
SDValue SubVal = SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, |
| 2142 |
MVT::i32, V, SubRegIdx), 0); |
2142 |
MVT::i32, V, SubRegIdx), 0); |
| 2143 |
return SubVal; |
2143 |
return SubVal; |
| 2144 |
} |
2144 |
} |
| 2145 |
|
2145 |
|
| 2146 |
// Depending on the number of groups for a particular value, it might be |
2146 |
// Depending on the number of groups for a particular value, it might be |
| 2147 |
// better to rotate, mask explicitly (using andi/andis), and then or the |
2147 |
// better to rotate, mask explicitly (using andi/andis), and then or the |
| 2148 |
// result. Select this part of the result first. |
2148 |
// result. Select this part of the result first. |
| 2149 |
void SelectAndParts32(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) { |
2149 |
void SelectAndParts32(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) { |
| 2150 |
if (BPermRewriterNoMasking) |
2150 |
if (BPermRewriterNoMasking) |
| 2151 |
return; |
2151 |
return; |
| 2152 |
|
2152 |
|
| 2153 |
for (ValueRotInfo &VRI : ValueRotsVec) { |
2153 |
for (ValueRotInfo &VRI : ValueRotsVec) { |
| 2154 |
unsigned Mask = 0; |
2154 |
unsigned Mask = 0; |
| 2155 |
for (unsigned i = 0; i < Bits.size(); ++i) { |
2155 |
for (unsigned i = 0; i < Bits.size(); ++i) { |
| 2156 |
if (!Bits[i].hasValue() || Bits[i].getValue() != VRI.V) |
2156 |
if (!Bits[i].hasValue() || Bits[i].getValue() != VRI.V) |
| 2157 |
continue; |
2157 |
continue; |
| 2158 |
if (RLAmt[i] != VRI.RLAmt) |
2158 |
if (RLAmt[i] != VRI.RLAmt) |
| 2159 |
continue; |
2159 |
continue; |
| 2160 |
Mask |= (1u << i); |
2160 |
Mask |= (1u << i); |
| 2161 |
} |
2161 |
} |
| 2162 |
|
2162 |
|
| 2163 |
// Compute the masks for andi/andis that would be necessary. |
2163 |
// Compute the masks for andi/andis that would be necessary. |
| 2164 |
unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16; |
2164 |
unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16; |
| 2165 |
assert((ANDIMask != 0 || ANDISMask != 0) && |
2165 |
assert((ANDIMask != 0 || ANDISMask != 0) && |
| 2166 |
"No set bits in mask for value bit groups"); |
2166 |
"No set bits in mask for value bit groups"); |
| 2167 |
bool NeedsRotate = VRI.RLAmt != 0; |
2167 |
bool NeedsRotate = VRI.RLAmt != 0; |
| 2168 |
|
2168 |
|
| 2169 |
// We're trying to minimize the number of instructions. If we have one |
2169 |
// We're trying to minimize the number of instructions. If we have one |
| 2170 |
// group, using one of andi/andis can break even. If we have three |
2170 |
// group, using one of andi/andis can break even. If we have three |
| 2171 |
// groups, we can use both andi and andis and break even (to use both |
2171 |
// groups, we can use both andi and andis and break even (to use both |
| 2172 |
// andi and andis we also need to or the results together). We need four |
2172 |
// andi and andis we also need to or the results together). We need four |
| 2173 |
// groups if we also need to rotate. To use andi/andis we need to do more |
2173 |
// groups if we also need to rotate. To use andi/andis we need to do more |
| 2174 |
// than break even because rotate-and-mask instructions tend to be easier |
2174 |
// than break even because rotate-and-mask instructions tend to be easier |
| 2175 |
// to schedule. |
2175 |
// to schedule. |
| 2176 |
|
2176 |
|
| 2177 |
// FIXME: We've biased here against using andi/andis, which is right for |
2177 |
// FIXME: We've biased here against using andi/andis, which is right for |
| 2178 |
// POWER cores, but not optimal everywhere. For example, on the A2, |
2178 |
// POWER cores, but not optimal everywhere. For example, on the A2, |
| 2179 |
// andi/andis have single-cycle latency whereas the rotate-and-mask |
2179 |
// andi/andis have single-cycle latency whereas the rotate-and-mask |
| 2180 |
// instructions take two cycles, and it would be better to bias toward |
2180 |
// instructions take two cycles, and it would be better to bias toward |
| 2181 |
// andi/andis in break-even cases. |
2181 |
// andi/andis in break-even cases. |
| 2182 |
|
2182 |
|
| 2183 |
unsigned NumAndInsts = (unsigned) NeedsRotate + |
2183 |
unsigned NumAndInsts = (unsigned) NeedsRotate + |
| 2184 |
(unsigned) (ANDIMask != 0) + |
2184 |
(unsigned) (ANDIMask != 0) + |
| 2185 |
(unsigned) (ANDISMask != 0) + |
2185 |
(unsigned) (ANDISMask != 0) + |
| 2186 |
(unsigned) (ANDIMask != 0 && ANDISMask != 0) + |
2186 |
(unsigned) (ANDIMask != 0 && ANDISMask != 0) + |
| 2187 |
(unsigned) (bool) Res; |
2187 |
(unsigned) (bool) Res; |
| 2188 |
|
2188 |
|
| 2189 |
LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode() |
2189 |
LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode() |
| 2190 |
<< " RL: " << VRI.RLAmt << ":" |
2190 |
<< " RL: " << VRI.RLAmt << ":" |
| 2191 |
<< "\n\t\t\tisel using masking: " << NumAndInsts |
2191 |
<< "\n\t\t\tisel using masking: " << NumAndInsts |
| 2192 |
<< " using rotates: " << VRI.NumGroups << "\n"); |
2192 |
<< " using rotates: " << VRI.NumGroups << "\n"); |
| 2193 |
|
2193 |
|
| 2194 |
if (NumAndInsts >= VRI.NumGroups) |
2194 |
if (NumAndInsts >= VRI.NumGroups) |
| 2195 |
continue; |
2195 |
continue; |
| 2196 |
|
2196 |
|
| 2197 |
LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n"); |
2197 |
LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n"); |
| 2198 |
|
2198 |
|
| 2199 |
if (InstCnt) *InstCnt += NumAndInsts; |
2199 |
if (InstCnt) *InstCnt += NumAndInsts; |
| 2200 |
|
2200 |
|
| 2201 |
SDValue VRot; |
2201 |
SDValue VRot; |
| 2202 |
if (VRI.RLAmt) { |
2202 |
if (VRI.RLAmt) { |
| 2203 |
SDValue Ops[] = |
2203 |
SDValue Ops[] = |
| 2204 |
{ TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl), |
2204 |
{ TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl), |
| 2205 |
getI32Imm(0, dl), getI32Imm(31, dl) }; |
2205 |
getI32Imm(0, dl), getI32Imm(31, dl) }; |
| 2206 |
VRot = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, |
2206 |
VRot = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, |
| 2207 |
Ops), 0); |
2207 |
Ops), 0); |
| 2208 |
} else { |
2208 |
} else { |
| 2209 |
VRot = TruncateToInt32(VRI.V, dl); |
2209 |
VRot = TruncateToInt32(VRI.V, dl); |
| 2210 |
} |
2210 |
} |
| 2211 |
|
2211 |
|
| 2212 |
SDValue ANDIVal, ANDISVal; |
2212 |
SDValue ANDIVal, ANDISVal; |
| 2213 |
if (ANDIMask != 0) |
2213 |
if (ANDIMask != 0) |
| 2214 |
ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32, |
2214 |
ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32, |
| 2215 |
VRot, getI32Imm(ANDIMask, dl)), |
2215 |
VRot, getI32Imm(ANDIMask, dl)), |
| 2216 |
0); |
2216 |
0); |
| 2217 |
if (ANDISMask != 0) |
2217 |
if (ANDISMask != 0) |
| 2218 |
ANDISVal = |
2218 |
ANDISVal = |
| 2219 |
SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, VRot, |
2219 |
SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, VRot, |
| 2220 |
getI32Imm(ANDISMask, dl)), |
2220 |
getI32Imm(ANDISMask, dl)), |
| 2221 |
0); |
2221 |
0); |
| 2222 |
|
2222 |
|
| 2223 |
SDValue TotalVal; |
2223 |
SDValue TotalVal; |
| 2224 |
if (!ANDIVal) |
2224 |
if (!ANDIVal) |
| 2225 |
TotalVal = ANDISVal; |
2225 |
TotalVal = ANDISVal; |
| 2226 |
else if (!ANDISVal) |
2226 |
else if (!ANDISVal) |
| 2227 |
TotalVal = ANDIVal; |
2227 |
TotalVal = ANDIVal; |
| 2228 |
else |
2228 |
else |
| 2229 |
TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, |
2229 |
TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, |
| 2230 |
ANDIVal, ANDISVal), 0); |
2230 |
ANDIVal, ANDISVal), 0); |
| 2231 |
|
2231 |
|
| 2232 |
if (!Res) |
2232 |
if (!Res) |
| 2233 |
Res = TotalVal; |
2233 |
Res = TotalVal; |
| 2234 |
else |
2234 |
else |
| 2235 |
Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, |
2235 |
Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, |
| 2236 |
Res, TotalVal), 0); |
2236 |
Res, TotalVal), 0); |
| 2237 |
|
2237 |
|
| 2238 |
// Now, remove all groups with this underlying value and rotation |
2238 |
// Now, remove all groups with this underlying value and rotation |
| 2239 |
// factor. |
2239 |
// factor. |
| 2240 |
eraseMatchingBitGroups([VRI](const BitGroup &BG) { |
2240 |
eraseMatchingBitGroups([VRI](const BitGroup &BG) { |
| 2241 |
return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt; |
2241 |
return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt; |
| 2242 |
}); |
2242 |
}); |
| 2243 |
} |
2243 |
} |
| 2244 |
} |
2244 |
} |
| 2245 |
|
2245 |
|
| 2246 |
// Instruction selection for the 32-bit case. |
2246 |
// Instruction selection for the 32-bit case. |
| 2247 |
SDNode *Select32(SDNode *N, bool LateMask, unsigned *InstCnt) { |
2247 |
SDNode *Select32(SDNode *N, bool LateMask, unsigned *InstCnt) { |
| 2248 |
SDLoc dl(N); |
2248 |
SDLoc dl(N); |
| 2249 |
SDValue Res; |
2249 |
SDValue Res; |
| 2250 |
|
2250 |
|
| 2251 |
if (InstCnt) *InstCnt = 0; |
2251 |
if (InstCnt) *InstCnt = 0; |
| 2252 |
|
2252 |
|
| 2253 |
// Take care of cases that should use andi/andis first. |
2253 |
// Take care of cases that should use andi/andis first. |
| 2254 |
SelectAndParts32(dl, Res, InstCnt); |
2254 |
SelectAndParts32(dl, Res, InstCnt); |
| 2255 |
|
2255 |
|
| 2256 |
// If we've not yet selected a 'starting' instruction, and we have no zeros |
2256 |
// If we've not yet selected a 'starting' instruction, and we have no zeros |
| 2257 |
// to fill in, select the (Value, RLAmt) with the highest priority (largest |
2257 |
// to fill in, select the (Value, RLAmt) with the highest priority (largest |
| 2258 |
// number of groups), and start with this rotated value. |
2258 |
// number of groups), and start with this rotated value. |
| 2259 |
if ((!NeedMask || LateMask) && !Res) { |
2259 |
if ((!NeedMask || LateMask) && !Res) { |
| 2260 |
ValueRotInfo &VRI = ValueRotsVec[0]; |
2260 |
ValueRotInfo &VRI = ValueRotsVec[0]; |
| 2261 |
if (VRI.RLAmt) { |
2261 |
if (VRI.RLAmt) { |
| 2262 |
if (InstCnt) *InstCnt += 1; |
2262 |
if (InstCnt) *InstCnt += 1; |
| 2263 |
SDValue Ops[] = |
2263 |
SDValue Ops[] = |
| 2264 |
{ TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl), |
2264 |
{ TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl), |
| 2265 |
getI32Imm(0, dl), getI32Imm(31, dl) }; |
2265 |
getI32Imm(0, dl), getI32Imm(31, dl) }; |
| 2266 |
Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), |
2266 |
Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), |
| 2267 |
0); |
2267 |
0); |
| 2268 |
} else { |
2268 |
} else { |
| 2269 |
Res = TruncateToInt32(VRI.V, dl); |
2269 |
Res = TruncateToInt32(VRI.V, dl); |
| 2270 |
} |
2270 |
} |
| 2271 |
|
2271 |
|
| 2272 |
// Now, remove all groups with this underlying value and rotation factor. |
2272 |
// Now, remove all groups with this underlying value and rotation factor. |
| 2273 |
eraseMatchingBitGroups([VRI](const BitGroup &BG) { |
2273 |
eraseMatchingBitGroups([VRI](const BitGroup &BG) { |
| 2274 |
return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt; |
2274 |
return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt; |
| 2275 |
}); |
2275 |
}); |
| 2276 |
} |
2276 |
} |
| 2277 |
|
2277 |
|
| 2278 |
if (InstCnt) *InstCnt += BitGroups.size(); |
2278 |
if (InstCnt) *InstCnt += BitGroups.size(); |
| 2279 |
|
2279 |
|
| 2280 |
// Insert the other groups (one at a time). |
2280 |
// Insert the other groups (one at a time). |
| 2281 |
for (auto &BG : BitGroups) { |
2281 |
for (auto &BG : BitGroups) { |
| 2282 |
if (!Res) { |
2282 |
if (!Res) { |
| 2283 |
SDValue Ops[] = |
2283 |
SDValue Ops[] = |
| 2284 |
{ TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl), |
2284 |
{ TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl), |
| 2285 |
getI32Imm(Bits.size() - BG.EndIdx - 1, dl), |
2285 |
getI32Imm(Bits.size() - BG.EndIdx - 1, dl), |
| 2286 |
getI32Imm(Bits.size() - BG.StartIdx - 1, dl) }; |
2286 |
getI32Imm(Bits.size() - BG.StartIdx - 1, dl) }; |
| 2287 |
Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); |
2287 |
Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); |
| 2288 |
} else { |
2288 |
} else { |
| 2289 |
SDValue Ops[] = |
2289 |
SDValue Ops[] = |
| 2290 |
{ Res, TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl), |
2290 |
{ Res, TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl), |
| 2291 |
getI32Imm(Bits.size() - BG.EndIdx - 1, dl), |
2291 |
getI32Imm(Bits.size() - BG.EndIdx - 1, dl), |
| 2292 |
getI32Imm(Bits.size() - BG.StartIdx - 1, dl) }; |
2292 |
getI32Imm(Bits.size() - BG.StartIdx - 1, dl) }; |
| 2293 |
Res = SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops), 0); |
2293 |
Res = SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops), 0); |
| 2294 |
} |
2294 |
} |
| 2295 |
} |
2295 |
} |
| 2296 |
|
2296 |
|
| 2297 |
if (LateMask) { |
2297 |
if (LateMask) { |
| 2298 |
unsigned Mask = (unsigned) getZerosMask(); |
2298 |
unsigned Mask = (unsigned) getZerosMask(); |
| 2299 |
|
2299 |
|
| 2300 |
unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16; |
2300 |
unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16; |
| 2301 |
assert((ANDIMask != 0 || ANDISMask != 0) && |
2301 |
assert((ANDIMask != 0 || ANDISMask != 0) && |
| 2302 |
"No set bits in zeros mask?"); |
2302 |
"No set bits in zeros mask?"); |
| 2303 |
|
2303 |
|
| 2304 |
if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) + |
2304 |
if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) + |
| 2305 |
(unsigned) (ANDISMask != 0) + |
2305 |
(unsigned) (ANDISMask != 0) + |
| 2306 |
(unsigned) (ANDIMask != 0 && ANDISMask != 0); |
2306 |
(unsigned) (ANDIMask != 0 && ANDISMask != 0); |
| 2307 |
|
2307 |
|
| 2308 |
SDValue ANDIVal, ANDISVal; |
2308 |
SDValue ANDIVal, ANDISVal; |
| 2309 |
if (ANDIMask != 0) |
2309 |
if (ANDIMask != 0) |
| 2310 |
ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32, |
2310 |
ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32, |
| 2311 |
Res, getI32Imm(ANDIMask, dl)), |
2311 |
Res, getI32Imm(ANDIMask, dl)), |
| 2312 |
0); |
2312 |
0); |
| 2313 |
if (ANDISMask != 0) |
2313 |
if (ANDISMask != 0) |
| 2314 |
ANDISVal = |
2314 |
ANDISVal = |
| 2315 |
SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, Res, |
2315 |
SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, Res, |
| 2316 |
getI32Imm(ANDISMask, dl)), |
2316 |
getI32Imm(ANDISMask, dl)), |
| 2317 |
0); |
2317 |
0); |
| 2318 |
|
2318 |
|
| 2319 |
if (!ANDIVal) |
2319 |
if (!ANDIVal) |
| 2320 |
Res = ANDISVal; |
2320 |
Res = ANDISVal; |
| 2321 |
else if (!ANDISVal) |
2321 |
else if (!ANDISVal) |
| 2322 |
Res = ANDIVal; |
2322 |
Res = ANDIVal; |
| 2323 |
else |
2323 |
else |
| 2324 |
Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, |
2324 |
Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, |
| 2325 |
ANDIVal, ANDISVal), 0); |
2325 |
ANDIVal, ANDISVal), 0); |
| 2326 |
} |
2326 |
} |
| 2327 |
|
2327 |
|
| 2328 |
return Res.getNode(); |
2328 |
return Res.getNode(); |
| 2329 |
} |
2329 |
} |
| 2330 |
|
2330 |
|
| 2331 |
unsigned SelectRotMask64Count(unsigned RLAmt, bool Repl32, |
2331 |
unsigned SelectRotMask64Count(unsigned RLAmt, bool Repl32, |
| 2332 |
unsigned MaskStart, unsigned MaskEnd, |
2332 |
unsigned MaskStart, unsigned MaskEnd, |
| 2333 |
bool IsIns) { |
2333 |
bool IsIns) { |
| 2334 |
// In the notation used by the instructions, 'start' and 'end' are reversed |
2334 |
// In the notation used by the instructions, 'start' and 'end' are reversed |
| 2335 |
// because bits are counted from high to low order. |
2335 |
// because bits are counted from high to low order. |
| 2336 |
unsigned InstMaskStart = 64 - MaskEnd - 1, |
2336 |
unsigned InstMaskStart = 64 - MaskEnd - 1, |
| 2337 |
InstMaskEnd = 64 - MaskStart - 1; |
2337 |
InstMaskEnd = 64 - MaskStart - 1; |
| 2338 |
|
2338 |
|
| 2339 |
if (Repl32) |
2339 |
if (Repl32) |
| 2340 |
return 1; |
2340 |
return 1; |
| 2341 |
|
2341 |
|
| 2342 |
if ((!IsIns && (InstMaskEnd == 63 || InstMaskStart == 0)) || |
2342 |
if ((!IsIns && (InstMaskEnd == 63 || InstMaskStart == 0)) || |
| 2343 |
InstMaskEnd == 63 - RLAmt) |
2343 |
InstMaskEnd == 63 - RLAmt) |
| 2344 |
return 1; |
2344 |
return 1; |
| 2345 |
|
2345 |
|
| 2346 |
return 2; |
2346 |
return 2; |
| 2347 |
} |
2347 |
} |
| 2348 |
|
2348 |
|
| 2349 |
// For 64-bit values, not all combinations of rotates and masks are |
2349 |
// For 64-bit values, not all combinations of rotates and masks are |
| 2350 |
// available. Produce one if it is available. |
2350 |
// available. Produce one if it is available. |
| 2351 |
SDValue SelectRotMask64(SDValue V, const SDLoc &dl, unsigned RLAmt, |
2351 |
SDValue SelectRotMask64(SDValue V, const SDLoc &dl, unsigned RLAmt, |
| 2352 |
bool Repl32, unsigned MaskStart, unsigned MaskEnd, |
2352 |
bool Repl32, unsigned MaskStart, unsigned MaskEnd, |
| 2353 |
unsigned *InstCnt = nullptr) { |
2353 |
unsigned *InstCnt = nullptr) { |
| 2354 |
// In the notation used by the instructions, 'start' and 'end' are reversed |
2354 |
// In the notation used by the instructions, 'start' and 'end' are reversed |
| 2355 |
// because bits are counted from high to low order. |
2355 |
// because bits are counted from high to low order. |
| 2356 |
unsigned InstMaskStart = 64 - MaskEnd - 1, |
2356 |
unsigned InstMaskStart = 64 - MaskEnd - 1, |
| 2357 |
InstMaskEnd = 64 - MaskStart - 1; |
2357 |
InstMaskEnd = 64 - MaskStart - 1; |
| 2358 |
|
2358 |
|
| 2359 |
if (InstCnt) *InstCnt += 1; |
2359 |
if (InstCnt) *InstCnt += 1; |
| 2360 |
|
2360 |
|
| 2361 |
if (Repl32) { |
2361 |
if (Repl32) { |
| 2362 |
// This rotation amount assumes that the lower 32 bits of the quantity |
2362 |
// This rotation amount assumes that the lower 32 bits of the quantity |
| 2363 |
// are replicated in the high 32 bits by the rotation operator (which is |
2363 |
// are replicated in the high 32 bits by the rotation operator (which is |
| 2364 |
// done by rlwinm and friends). |
2364 |
// done by rlwinm and friends). |
| 2365 |
assert(InstMaskStart >= 32 && "Mask cannot start out of range"); |
2365 |
assert(InstMaskStart >= 32 && "Mask cannot start out of range"); |
| 2366 |
assert(InstMaskEnd >= 32 && "Mask cannot end out of range"); |
2366 |
assert(InstMaskEnd >= 32 && "Mask cannot end out of range"); |
| 2367 |
SDValue Ops[] = |
2367 |
SDValue Ops[] = |
| 2368 |
{ ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), |
2368 |
{ ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), |
| 2369 |
getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) }; |
2369 |
getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) }; |
| 2370 |
return SDValue(CurDAG->getMachineNode(PPC::RLWINM8, dl, MVT::i64, |
2370 |
return SDValue(CurDAG->getMachineNode(PPC::RLWINM8, dl, MVT::i64, |
| 2371 |
Ops), 0); |
2371 |
Ops), 0); |
| 2372 |
} |
2372 |
} |
| 2373 |
|
2373 |
|
| 2374 |
if (InstMaskEnd == 63) { |
2374 |
if (InstMaskEnd == 63) { |
| 2375 |
SDValue Ops[] = |
2375 |
SDValue Ops[] = |
| 2376 |
{ ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), |
2376 |
{ ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), |
| 2377 |
getI32Imm(InstMaskStart, dl) }; |
2377 |
getI32Imm(InstMaskStart, dl) }; |
| 2378 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Ops), 0); |
2378 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Ops), 0); |
| 2379 |
} |
2379 |
} |
| 2380 |
|
2380 |
|
| 2381 |
if (InstMaskStart == 0) { |
2381 |
if (InstMaskStart == 0) { |
| 2382 |
SDValue Ops[] = |
2382 |
SDValue Ops[] = |
| 2383 |
{ ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), |
2383 |
{ ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), |
| 2384 |
getI32Imm(InstMaskEnd, dl) }; |
2384 |
getI32Imm(InstMaskEnd, dl) }; |
| 2385 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Ops), 0); |
2385 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Ops), 0); |
| 2386 |
} |
2386 |
} |
| 2387 |
|
2387 |
|
| 2388 |
if (InstMaskEnd == 63 - RLAmt) { |
2388 |
if (InstMaskEnd == 63 - RLAmt) { |
| 2389 |
SDValue Ops[] = |
2389 |
SDValue Ops[] = |
| 2390 |
{ ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), |
2390 |
{ ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), |
| 2391 |
getI32Imm(InstMaskStart, dl) }; |
2391 |
getI32Imm(InstMaskStart, dl) }; |
| 2392 |
return SDValue(CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, Ops), 0); |
2392 |
return SDValue(CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, Ops), 0); |
| 2393 |
} |
2393 |
} |
| 2394 |
|
2394 |
|
| 2395 |
// We cannot do this with a single instruction, so we'll use two. The |
2395 |
// We cannot do this with a single instruction, so we'll use two. The |
| 2396 |
// problem is that we're not free to choose both a rotation amount and mask |
2396 |
// problem is that we're not free to choose both a rotation amount and mask |
| 2397 |
// start and end independently. We can choose an arbitrary mask start and |
2397 |
// start and end independently. We can choose an arbitrary mask start and |
| 2398 |
// end, but then the rotation amount is fixed. Rotation, however, can be |
2398 |
// end, but then the rotation amount is fixed. Rotation, however, can be |
| 2399 |
// inverted, and so by applying an "inverse" rotation first, we can get the |
2399 |
// inverted, and so by applying an "inverse" rotation first, we can get the |
| 2400 |
// desired result. |
2400 |
// desired result. |
| 2401 |
if (InstCnt) *InstCnt += 1; |
2401 |
if (InstCnt) *InstCnt += 1; |
| 2402 |
|
2402 |
|
| 2403 |
// The rotation mask for the second instruction must be MaskStart. |
2403 |
// The rotation mask for the second instruction must be MaskStart. |
| 2404 |
unsigned RLAmt2 = MaskStart; |
2404 |
unsigned RLAmt2 = MaskStart; |
| 2405 |
// The first instruction must rotate V so that the overall rotation amount |
2405 |
// The first instruction must rotate V so that the overall rotation amount |
| 2406 |
// is RLAmt. |
2406 |
// is RLAmt. |
| 2407 |
unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64; |
2407 |
unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64; |
| 2408 |
if (RLAmt1) |
2408 |
if (RLAmt1) |
| 2409 |
V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63); |
2409 |
V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63); |
| 2410 |
return SelectRotMask64(V, dl, RLAmt2, false, MaskStart, MaskEnd); |
2410 |
return SelectRotMask64(V, dl, RLAmt2, false, MaskStart, MaskEnd); |
| 2411 |
} |
2411 |
} |
| 2412 |
|
2412 |
|
| 2413 |
// For 64-bit values, not all combinations of rotates and masks are |
2413 |
// For 64-bit values, not all combinations of rotates and masks are |
| 2414 |
// available. Produce a rotate-mask-and-insert if one is available. |
2414 |
// available. Produce a rotate-mask-and-insert if one is available. |
| 2415 |
SDValue SelectRotMaskIns64(SDValue Base, SDValue V, const SDLoc &dl, |
2415 |
SDValue SelectRotMaskIns64(SDValue Base, SDValue V, const SDLoc &dl, |
| 2416 |
unsigned RLAmt, bool Repl32, unsigned MaskStart, |
2416 |
unsigned RLAmt, bool Repl32, unsigned MaskStart, |
| 2417 |
unsigned MaskEnd, unsigned *InstCnt = nullptr) { |
2417 |
unsigned MaskEnd, unsigned *InstCnt = nullptr) { |
| 2418 |
// In the notation used by the instructions, 'start' and 'end' are reversed |
2418 |
// In the notation used by the instructions, 'start' and 'end' are reversed |
| 2419 |
// because bits are counted from high to low order. |
2419 |
// because bits are counted from high to low order. |
| 2420 |
unsigned InstMaskStart = 64 - MaskEnd - 1, |
2420 |
unsigned InstMaskStart = 64 - MaskEnd - 1, |
| 2421 |
InstMaskEnd = 64 - MaskStart - 1; |
2421 |
InstMaskEnd = 64 - MaskStart - 1; |
| 2422 |
|
2422 |
|
| 2423 |
if (InstCnt) *InstCnt += 1; |
2423 |
if (InstCnt) *InstCnt += 1; |
| 2424 |
|
2424 |
|
| 2425 |
if (Repl32) { |
2425 |
if (Repl32) { |
| 2426 |
// This rotation amount assumes that the lower 32 bits of the quantity |
2426 |
// This rotation amount assumes that the lower 32 bits of the quantity |
| 2427 |
// are replicated in the high 32 bits by the rotation operator (which is |
2427 |
// are replicated in the high 32 bits by the rotation operator (which is |
| 2428 |
// done by rlwinm and friends). |
2428 |
// done by rlwinm and friends). |
| 2429 |
assert(InstMaskStart >= 32 && "Mask cannot start out of range"); |
2429 |
assert(InstMaskStart >= 32 && "Mask cannot start out of range"); |
| 2430 |
assert(InstMaskEnd >= 32 && "Mask cannot end out of range"); |
2430 |
assert(InstMaskEnd >= 32 && "Mask cannot end out of range"); |
| 2431 |
SDValue Ops[] = |
2431 |
SDValue Ops[] = |
| 2432 |
{ ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), |
2432 |
{ ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), |
| 2433 |
getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) }; |
2433 |
getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) }; |
| 2434 |
return SDValue(CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, |
2434 |
return SDValue(CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, |
| 2435 |
Ops), 0); |
2435 |
Ops), 0); |
| 2436 |
} |
2436 |
} |
| 2437 |
|
2437 |
|
| 2438 |
if (InstMaskEnd == 63 - RLAmt) { |
2438 |
if (InstMaskEnd == 63 - RLAmt) { |
| 2439 |
SDValue Ops[] = |
2439 |
SDValue Ops[] = |
| 2440 |
{ ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), |
2440 |
{ ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), |
| 2441 |
getI32Imm(InstMaskStart, dl) }; |
2441 |
getI32Imm(InstMaskStart, dl) }; |
| 2442 |
return SDValue(CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops), 0); |
2442 |
return SDValue(CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops), 0); |
| 2443 |
} |
2443 |
} |
| 2444 |
|
2444 |
|
| 2445 |
// We cannot do this with a single instruction, so we'll use two. The |
2445 |
// We cannot do this with a single instruction, so we'll use two. The |
| 2446 |
// problem is that we're not free to choose both a rotation amount and mask |
2446 |
// problem is that we're not free to choose both a rotation amount and mask |
| 2447 |
// start and end independently. We can choose an arbitrary mask start and |
2447 |
// start and end independently. We can choose an arbitrary mask start and |
| 2448 |
// end, but then the rotation amount is fixed. Rotation, however, can be |
2448 |
// end, but then the rotation amount is fixed. Rotation, however, can be |
| 2449 |
// inverted, and so by applying an "inverse" rotation first, we can get the |
2449 |
// inverted, and so by applying an "inverse" rotation first, we can get the |
| 2450 |
// desired result. |
2450 |
// desired result. |
| 2451 |
if (InstCnt) *InstCnt += 1; |
2451 |
if (InstCnt) *InstCnt += 1; |
| 2452 |
|
2452 |
|
| 2453 |
// The rotation mask for the second instruction must be MaskStart. |
2453 |
// The rotation mask for the second instruction must be MaskStart. |
| 2454 |
unsigned RLAmt2 = MaskStart; |
2454 |
unsigned RLAmt2 = MaskStart; |
| 2455 |
// The first instruction must rotate V so that the overall rotation amount |
2455 |
// The first instruction must rotate V so that the overall rotation amount |
| 2456 |
// is RLAmt. |
2456 |
// is RLAmt. |
| 2457 |
unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64; |
2457 |
unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64; |
| 2458 |
if (RLAmt1) |
2458 |
if (RLAmt1) |
| 2459 |
V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63); |
2459 |
V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63); |
| 2460 |
return SelectRotMaskIns64(Base, V, dl, RLAmt2, false, MaskStart, MaskEnd); |
2460 |
return SelectRotMaskIns64(Base, V, dl, RLAmt2, false, MaskStart, MaskEnd); |
| 2461 |
} |
2461 |
} |
| 2462 |
|
2462 |
|
| 2463 |
void SelectAndParts64(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) { |
2463 |
void SelectAndParts64(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) { |
| 2464 |
if (BPermRewriterNoMasking) |
2464 |
if (BPermRewriterNoMasking) |
| 2465 |
return; |
2465 |
return; |
| 2466 |
|
2466 |
|
| 2467 |
// The idea here is the same as in the 32-bit version, but with additional |
2467 |
// The idea here is the same as in the 32-bit version, but with additional |
| 2468 |
// complications from the fact that Repl32 might be true. Because we |
2468 |
// complications from the fact that Repl32 might be true. Because we |
| 2469 |
// aggressively convert bit groups to Repl32 form (which, for small |
2469 |
// aggressively convert bit groups to Repl32 form (which, for small |
| 2470 |
// rotation factors, involves no other change), and then coalesce, it might |
2470 |
// rotation factors, involves no other change), and then coalesce, it might |
| 2471 |
// be the case that a single 64-bit masking operation could handle both |
2471 |
// be the case that a single 64-bit masking operation could handle both |
| 2472 |
// some Repl32 groups and some non-Repl32 groups. If converting to Repl32 |
2472 |
// some Repl32 groups and some non-Repl32 groups. If converting to Repl32 |
| 2473 |
// form allowed coalescing, then we must use a 32-bit rotaton in order to |
2473 |
// form allowed coalescing, then we must use a 32-bit rotaton in order to |
| 2474 |
// completely capture the new combined bit group. |
2474 |
// completely capture the new combined bit group. |
| 2475 |
|
2475 |
|
| 2476 |
for (ValueRotInfo &VRI : ValueRotsVec) { |
2476 |
for (ValueRotInfo &VRI : ValueRotsVec) { |
| 2477 |
uint64_t Mask = 0; |
2477 |
uint64_t Mask = 0; |
| 2478 |
|
2478 |
|
| 2479 |
// We need to add to the mask all bits from the associated bit groups. |
2479 |
// We need to add to the mask all bits from the associated bit groups. |
| 2480 |
// If Repl32 is false, we need to add bits from bit groups that have |
2480 |
// If Repl32 is false, we need to add bits from bit groups that have |
| 2481 |
// Repl32 true, but are trivially convertable to Repl32 false. Such a |
2481 |
// Repl32 true, but are trivially convertable to Repl32 false. Such a |
| 2482 |
// group is trivially convertable if it overlaps only with the lower 32 |
2482 |
// group is trivially convertable if it overlaps only with the lower 32 |
| 2483 |
// bits, and the group has not been coalesced. |
2483 |
// bits, and the group has not been coalesced. |
| 2484 |
auto MatchingBG = [VRI](const BitGroup &BG) { |
2484 |
auto MatchingBG = [VRI](const BitGroup &BG) { |
| 2485 |
if (VRI.V != BG.V) |
2485 |
if (VRI.V != BG.V) |
| 2486 |
return false; |
2486 |
return false; |
| 2487 |
|
2487 |
|
| 2488 |
unsigned EffRLAmt = BG.RLAmt; |
2488 |
unsigned EffRLAmt = BG.RLAmt; |
| 2489 |
if (!VRI.Repl32 && BG.Repl32) { |
2489 |
if (!VRI.Repl32 && BG.Repl32) { |
| 2490 |
if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx <= BG.EndIdx && |
2490 |
if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx <= BG.EndIdx && |
| 2491 |
!BG.Repl32Coalesced) { |
2491 |
!BG.Repl32Coalesced) { |
| 2492 |
if (BG.Repl32CR) |
2492 |
if (BG.Repl32CR) |
| 2493 |
EffRLAmt += 32; |
2493 |
EffRLAmt += 32; |
| 2494 |
} else { |
2494 |
} else { |
| 2495 |
return false; |
2495 |
return false; |
| 2496 |
} |
2496 |
} |
| 2497 |
} else if (VRI.Repl32 != BG.Repl32) { |
2497 |
} else if (VRI.Repl32 != BG.Repl32) { |
| 2498 |
return false; |
2498 |
return false; |
| 2499 |
} |
2499 |
} |
| 2500 |
|
2500 |
|
| 2501 |
return VRI.RLAmt == EffRLAmt; |
2501 |
return VRI.RLAmt == EffRLAmt; |
| 2502 |
}; |
2502 |
}; |
| 2503 |
|
2503 |
|
| 2504 |
for (auto &BG : BitGroups) { |
2504 |
for (auto &BG : BitGroups) { |
| 2505 |
if (!MatchingBG(BG)) |
2505 |
if (!MatchingBG(BG)) |
| 2506 |
continue; |
2506 |
continue; |
| 2507 |
|
2507 |
|
| 2508 |
if (BG.StartIdx <= BG.EndIdx) { |
2508 |
if (BG.StartIdx <= BG.EndIdx) { |
| 2509 |
for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) |
2509 |
for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) |
| 2510 |
Mask |= (UINT64_C(1) << i); |
2510 |
Mask |= (UINT64_C(1) << i); |
| 2511 |
} else { |
2511 |
} else { |
| 2512 |
for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) |
2512 |
for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) |
| 2513 |
Mask |= (UINT64_C(1) << i); |
2513 |
Mask |= (UINT64_C(1) << i); |
| 2514 |
for (unsigned i = 0; i <= BG.EndIdx; ++i) |
2514 |
for (unsigned i = 0; i <= BG.EndIdx; ++i) |
| 2515 |
Mask |= (UINT64_C(1) << i); |
2515 |
Mask |= (UINT64_C(1) << i); |
| 2516 |
} |
2516 |
} |
| 2517 |
} |
2517 |
} |
| 2518 |
|
2518 |
|
| 2519 |
// We can use the 32-bit andi/andis technique if the mask does not |
2519 |
// We can use the 32-bit andi/andis technique if the mask does not |
| 2520 |
// require any higher-order bits. This can save an instruction compared |
2520 |
// require any higher-order bits. This can save an instruction compared |
| 2521 |
// to always using the general 64-bit technique. |
2521 |
// to always using the general 64-bit technique. |
| 2522 |
bool Use32BitInsts = isUInt<32>(Mask); |
2522 |
bool Use32BitInsts = isUInt<32>(Mask); |
| 2523 |
// Compute the masks for andi/andis that would be necessary. |
2523 |
// Compute the masks for andi/andis that would be necessary. |
| 2524 |
unsigned ANDIMask = (Mask & UINT16_MAX), |
2524 |
unsigned ANDIMask = (Mask & UINT16_MAX), |
| 2525 |
ANDISMask = (Mask >> 16) & UINT16_MAX; |
2525 |
ANDISMask = (Mask >> 16) & UINT16_MAX; |
| 2526 |
|
2526 |
|
| 2527 |
bool NeedsRotate = VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask)); |
2527 |
bool NeedsRotate = VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask)); |
| 2528 |
|
2528 |
|
| 2529 |
unsigned NumAndInsts = (unsigned) NeedsRotate + |
2529 |
unsigned NumAndInsts = (unsigned) NeedsRotate + |
| 2530 |
(unsigned) (bool) Res; |
2530 |
(unsigned) (bool) Res; |
| 2531 |
unsigned NumOfSelectInsts = 0; |
2531 |
unsigned NumOfSelectInsts = 0; |
| 2532 |
selectI64Imm(CurDAG, dl, Mask, &NumOfSelectInsts); |
2532 |
selectI64Imm(CurDAG, dl, Mask, &NumOfSelectInsts); |
| 2533 |
assert(NumOfSelectInsts > 0 && "Failed to select an i64 constant."); |
2533 |
assert(NumOfSelectInsts > 0 && "Failed to select an i64 constant."); |
| 2534 |
if (Use32BitInsts) |
2534 |
if (Use32BitInsts) |
| 2535 |
NumAndInsts += (unsigned) (ANDIMask != 0) + (unsigned) (ANDISMask != 0) + |
2535 |
NumAndInsts += (unsigned) (ANDIMask != 0) + (unsigned) (ANDISMask != 0) + |
| 2536 |
(unsigned) (ANDIMask != 0 && ANDISMask != 0); |
2536 |
(unsigned) (ANDIMask != 0 && ANDISMask != 0); |
| 2537 |
else |
2537 |
else |
| 2538 |
NumAndInsts += NumOfSelectInsts + /* and */ 1; |
2538 |
NumAndInsts += NumOfSelectInsts + /* and */ 1; |
| 2539 |
|
2539 |
|
| 2540 |
unsigned NumRLInsts = 0; |
2540 |
unsigned NumRLInsts = 0; |
| 2541 |
bool FirstBG = true; |
2541 |
bool FirstBG = true; |
| 2542 |
bool MoreBG = false; |
2542 |
bool MoreBG = false; |
| 2543 |
for (auto &BG : BitGroups) { |
2543 |
for (auto &BG : BitGroups) { |
| 2544 |
if (!MatchingBG(BG)) { |
2544 |
if (!MatchingBG(BG)) { |
| 2545 |
MoreBG = true; |
2545 |
MoreBG = true; |
| 2546 |
continue; |
2546 |
continue; |
| 2547 |
} |
2547 |
} |
| 2548 |
NumRLInsts += |
2548 |
NumRLInsts += |
| 2549 |
SelectRotMask64Count(BG.RLAmt, BG.Repl32, BG.StartIdx, BG.EndIdx, |
2549 |
SelectRotMask64Count(BG.RLAmt, BG.Repl32, BG.StartIdx, BG.EndIdx, |
| 2550 |
!FirstBG); |
2550 |
!FirstBG); |
| 2551 |
FirstBG = false; |
2551 |
FirstBG = false; |
| 2552 |
} |
2552 |
} |
| 2553 |
|
2553 |
|
| 2554 |
LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode() |
2554 |
LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode() |
| 2555 |
<< " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":") |
2555 |
<< " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":") |
| 2556 |
<< "\n\t\t\tisel using masking: " << NumAndInsts |
2556 |
<< "\n\t\t\tisel using masking: " << NumAndInsts |
| 2557 |
<< " using rotates: " << NumRLInsts << "\n"); |
2557 |
<< " using rotates: " << NumRLInsts << "\n"); |
| 2558 |
|
2558 |
|
| 2559 |
// When we'd use andi/andis, we bias toward using the rotates (andi only |
2559 |
// When we'd use andi/andis, we bias toward using the rotates (andi only |
| 2560 |
// has a record form, and is cracked on POWER cores). However, when using |
2560 |
// has a record form, and is cracked on POWER cores). However, when using |
| 2561 |
// general 64-bit constant formation, bias toward the constant form, |
2561 |
// general 64-bit constant formation, bias toward the constant form, |
| 2562 |
// because that exposes more opportunities for CSE. |
2562 |
// because that exposes more opportunities for CSE. |
| 2563 |
if (NumAndInsts > NumRLInsts) |
2563 |
if (NumAndInsts > NumRLInsts) |
| 2564 |
continue; |
2564 |
continue; |
| 2565 |
// When merging multiple bit groups, instruction or is used. |
2565 |
// When merging multiple bit groups, instruction or is used. |
| 2566 |
// But when rotate is used, rldimi can inert the rotated value into any |
2566 |
// But when rotate is used, rldimi can inert the rotated value into any |
| 2567 |
// register, so instruction or can be avoided. |
2567 |
// register, so instruction or can be avoided. |
| 2568 |
if ((Use32BitInsts || MoreBG) && NumAndInsts == NumRLInsts) |
2568 |
if ((Use32BitInsts || MoreBG) && NumAndInsts == NumRLInsts) |
| 2569 |
continue; |
2569 |
continue; |
| 2570 |
|
2570 |
|
| 2571 |
LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n"); |
2571 |
LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n"); |
| 2572 |
|
2572 |
|
| 2573 |
if (InstCnt) *InstCnt += NumAndInsts; |
2573 |
if (InstCnt) *InstCnt += NumAndInsts; |
| 2574 |
|
2574 |
|
| 2575 |
SDValue VRot; |
2575 |
SDValue VRot; |
| 2576 |
// We actually need to generate a rotation if we have a non-zero rotation |
2576 |
// We actually need to generate a rotation if we have a non-zero rotation |
| 2577 |
// factor or, in the Repl32 case, if we care about any of the |
2577 |
// factor or, in the Repl32 case, if we care about any of the |
| 2578 |
// higher-order replicated bits. In the latter case, we generate a mask |
2578 |
// higher-order replicated bits. In the latter case, we generate a mask |
| 2579 |
// backward so that it actually includes the entire 64 bits. |
2579 |
// backward so that it actually includes the entire 64 bits. |
| 2580 |
if (VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask))) |
2580 |
if (VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask))) |
| 2581 |
VRot = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32, |
2581 |
VRot = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32, |
| 2582 |
VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63); |
2582 |
VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63); |
| 2583 |
else |
2583 |
else |
| 2584 |
VRot = VRI.V; |
2584 |
VRot = VRI.V; |
| 2585 |
|
2585 |
|
| 2586 |
SDValue TotalVal; |
2586 |
SDValue TotalVal; |
| 2587 |
if (Use32BitInsts) { |
2587 |
if (Use32BitInsts) { |
| 2588 |
assert((ANDIMask != 0 || ANDISMask != 0) && |
2588 |
assert((ANDIMask != 0 || ANDISMask != 0) && |
| 2589 |
"No set bits in mask when using 32-bit ands for 64-bit value"); |
2589 |
"No set bits in mask when using 32-bit ands for 64-bit value"); |
| 2590 |
|
2590 |
|
| 2591 |
SDValue ANDIVal, ANDISVal; |
2591 |
SDValue ANDIVal, ANDISVal; |
| 2592 |
if (ANDIMask != 0) |
2592 |
if (ANDIMask != 0) |
| 2593 |
ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64, |
2593 |
ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64, |
| 2594 |
ExtendToInt64(VRot, dl), |
2594 |
ExtendToInt64(VRot, dl), |
| 2595 |
getI32Imm(ANDIMask, dl)), |
2595 |
getI32Imm(ANDIMask, dl)), |
| 2596 |
0); |
2596 |
0); |
| 2597 |
if (ANDISMask != 0) |
2597 |
if (ANDISMask != 0) |
| 2598 |
ANDISVal = |
2598 |
ANDISVal = |
| 2599 |
SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64, |
2599 |
SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64, |
| 2600 |
ExtendToInt64(VRot, dl), |
2600 |
ExtendToInt64(VRot, dl), |
| 2601 |
getI32Imm(ANDISMask, dl)), |
2601 |
getI32Imm(ANDISMask, dl)), |
| 2602 |
0); |
2602 |
0); |
| 2603 |
|
2603 |
|
| 2604 |
if (!ANDIVal) |
2604 |
if (!ANDIVal) |
| 2605 |
TotalVal = ANDISVal; |
2605 |
TotalVal = ANDISVal; |
| 2606 |
else if (!ANDISVal) |
2606 |
else if (!ANDISVal) |
| 2607 |
TotalVal = ANDIVal; |
2607 |
TotalVal = ANDIVal; |
| 2608 |
else |
2608 |
else |
| 2609 |
TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, |
2609 |
TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, |
| 2610 |
ExtendToInt64(ANDIVal, dl), ANDISVal), 0); |
2610 |
ExtendToInt64(ANDIVal, dl), ANDISVal), 0); |
| 2611 |
} else { |
2611 |
} else { |
| 2612 |
TotalVal = SDValue(selectI64Imm(CurDAG, dl, Mask), 0); |
2612 |
TotalVal = SDValue(selectI64Imm(CurDAG, dl, Mask), 0); |
| 2613 |
TotalVal = |
2613 |
TotalVal = |
| 2614 |
SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64, |
2614 |
SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64, |
| 2615 |
ExtendToInt64(VRot, dl), TotalVal), |
2615 |
ExtendToInt64(VRot, dl), TotalVal), |
| 2616 |
0); |
2616 |
0); |
| 2617 |
} |
2617 |
} |
| 2618 |
|
2618 |
|
| 2619 |
if (!Res) |
2619 |
if (!Res) |
| 2620 |
Res = TotalVal; |
2620 |
Res = TotalVal; |
| 2621 |
else |
2621 |
else |
| 2622 |
Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, |
2622 |
Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, |
| 2623 |
ExtendToInt64(Res, dl), TotalVal), |
2623 |
ExtendToInt64(Res, dl), TotalVal), |
| 2624 |
0); |
2624 |
0); |
| 2625 |
|
2625 |
|
| 2626 |
// Now, remove all groups with this underlying value and rotation |
2626 |
// Now, remove all groups with this underlying value and rotation |
| 2627 |
// factor. |
2627 |
// factor. |
| 2628 |
eraseMatchingBitGroups(MatchingBG); |
2628 |
eraseMatchingBitGroups(MatchingBG); |
| 2629 |
} |
2629 |
} |
| 2630 |
} |
2630 |
} |
| 2631 |
|
2631 |
|
| 2632 |
// Instruction selection for the 64-bit case. |
2632 |
// Instruction selection for the 64-bit case. |
| 2633 |
SDNode *Select64(SDNode *N, bool LateMask, unsigned *InstCnt) { |
2633 |
SDNode *Select64(SDNode *N, bool LateMask, unsigned *InstCnt) { |
| 2634 |
SDLoc dl(N); |
2634 |
SDLoc dl(N); |
| 2635 |
SDValue Res; |
2635 |
SDValue Res; |
| 2636 |
|
2636 |
|
| 2637 |
if (InstCnt) *InstCnt = 0; |
2637 |
if (InstCnt) *InstCnt = 0; |
| 2638 |
|
2638 |
|
| 2639 |
// Take care of cases that should use andi/andis first. |
2639 |
// Take care of cases that should use andi/andis first. |
| 2640 |
SelectAndParts64(dl, Res, InstCnt); |
2640 |
SelectAndParts64(dl, Res, InstCnt); |
| 2641 |
|
2641 |
|
| 2642 |
// If we've not yet selected a 'starting' instruction, and we have no zeros |
2642 |
// If we've not yet selected a 'starting' instruction, and we have no zeros |
| 2643 |
// to fill in, select the (Value, RLAmt) with the highest priority (largest |
2643 |
// to fill in, select the (Value, RLAmt) with the highest priority (largest |
| 2644 |
// number of groups), and start with this rotated value. |
2644 |
// number of groups), and start with this rotated value. |
| 2645 |
if ((!NeedMask || LateMask) && !Res) { |
2645 |
if ((!NeedMask || LateMask) && !Res) { |
| 2646 |
// If we have both Repl32 groups and non-Repl32 groups, the non-Repl32 |
2646 |
// If we have both Repl32 groups and non-Repl32 groups, the non-Repl32 |
| 2647 |
// groups will come first, and so the VRI representing the largest number |
2647 |
// groups will come first, and so the VRI representing the largest number |
| 2648 |
// of groups might not be first (it might be the first Repl32 groups). |
2648 |
// of groups might not be first (it might be the first Repl32 groups). |
| 2649 |
unsigned MaxGroupsIdx = 0; |
2649 |
unsigned MaxGroupsIdx = 0; |
| 2650 |
if (!ValueRotsVec[0].Repl32) { |
2650 |
if (!ValueRotsVec[0].Repl32) { |
| 2651 |
for (unsigned i = 0, ie = ValueRotsVec.size(); i < ie; ++i) |
2651 |
for (unsigned i = 0, ie = ValueRotsVec.size(); i < ie; ++i) |
| 2652 |
if (ValueRotsVec[i].Repl32) { |
2652 |
if (ValueRotsVec[i].Repl32) { |
| 2653 |
if (ValueRotsVec[i].NumGroups > ValueRotsVec[0].NumGroups) |
2653 |
if (ValueRotsVec[i].NumGroups > ValueRotsVec[0].NumGroups) |
| 2654 |
MaxGroupsIdx = i; |
2654 |
MaxGroupsIdx = i; |
| 2655 |
break; |
2655 |
break; |
| 2656 |
} |
2656 |
} |
| 2657 |
} |
2657 |
} |
| 2658 |
|
2658 |
|
| 2659 |
ValueRotInfo &VRI = ValueRotsVec[MaxGroupsIdx]; |
2659 |
ValueRotInfo &VRI = ValueRotsVec[MaxGroupsIdx]; |
| 2660 |
bool NeedsRotate = false; |
2660 |
bool NeedsRotate = false; |
| 2661 |
if (VRI.RLAmt) { |
2661 |
if (VRI.RLAmt) { |
| 2662 |
NeedsRotate = true; |
2662 |
NeedsRotate = true; |
| 2663 |
} else if (VRI.Repl32) { |
2663 |
} else if (VRI.Repl32) { |
| 2664 |
for (auto &BG : BitGroups) { |
2664 |
for (auto &BG : BitGroups) { |
| 2665 |
if (BG.V != VRI.V || BG.RLAmt != VRI.RLAmt || |
2665 |
if (BG.V != VRI.V || BG.RLAmt != VRI.RLAmt || |
| 2666 |
BG.Repl32 != VRI.Repl32) |
2666 |
BG.Repl32 != VRI.Repl32) |
| 2667 |
continue; |
2667 |
continue; |
| 2668 |
|
2668 |
|
| 2669 |
// We don't need a rotate if the bit group is confined to the lower |
2669 |
// We don't need a rotate if the bit group is confined to the lower |
| 2670 |
// 32 bits. |
2670 |
// 32 bits. |
| 2671 |
if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx < BG.EndIdx) |
2671 |
if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx < BG.EndIdx) |
| 2672 |
continue; |
2672 |
continue; |
| 2673 |
|
2673 |
|
| 2674 |
NeedsRotate = true; |
2674 |
NeedsRotate = true; |
| 2675 |
break; |
2675 |
break; |
| 2676 |
} |
2676 |
} |
| 2677 |
} |
2677 |
} |
| 2678 |
|
2678 |
|
| 2679 |
if (NeedsRotate) |
2679 |
if (NeedsRotate) |
| 2680 |
Res = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32, |
2680 |
Res = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32, |
| 2681 |
VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63, |
2681 |
VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63, |
| 2682 |
InstCnt); |
2682 |
InstCnt); |
| 2683 |
else |
2683 |
else |
| 2684 |
Res = VRI.V; |
2684 |
Res = VRI.V; |
| 2685 |
|
2685 |
|
| 2686 |
// Now, remove all groups with this underlying value and rotation factor. |
2686 |
// Now, remove all groups with this underlying value and rotation factor. |
| 2687 |
if (Res) |
2687 |
if (Res) |
| 2688 |
eraseMatchingBitGroups([VRI](const BitGroup &BG) { |
2688 |
eraseMatchingBitGroups([VRI](const BitGroup &BG) { |
| 2689 |
return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt && |
2689 |
return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt && |
| 2690 |
BG.Repl32 == VRI.Repl32; |
2690 |
BG.Repl32 == VRI.Repl32; |
| 2691 |
}); |
2691 |
}); |
| 2692 |
} |
2692 |
} |
| 2693 |
|
2693 |
|
| 2694 |
// Because 64-bit rotates are more flexible than inserts, we might have a |
2694 |
// Because 64-bit rotates are more flexible than inserts, we might have a |
| 2695 |
// preference regarding which one we do first (to save one instruction). |
2695 |
// preference regarding which one we do first (to save one instruction). |
| 2696 |
if (!Res) |
2696 |
if (!Res) |
| 2697 |
for (auto I = BitGroups.begin(), IE = BitGroups.end(); I != IE; ++I) { |
2697 |
for (auto I = BitGroups.begin(), IE = BitGroups.end(); I != IE; ++I) { |
| 2698 |
if (SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx, |
2698 |
if (SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx, |
| 2699 |
false) < |
2699 |
false) < |
| 2700 |
SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx, |
2700 |
SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx, |
| 2701 |
true)) { |
2701 |
true)) { |
| 2702 |
if (I != BitGroups.begin()) { |
2702 |
if (I != BitGroups.begin()) { |
| 2703 |
BitGroup BG = *I; |
2703 |
BitGroup BG = *I; |
| 2704 |
BitGroups.erase(I); |
2704 |
BitGroups.erase(I); |
| 2705 |
BitGroups.insert(BitGroups.begin(), BG); |
2705 |
BitGroups.insert(BitGroups.begin(), BG); |
| 2706 |
} |
2706 |
} |
| 2707 |
|
2707 |
|
| 2708 |
break; |
2708 |
break; |
| 2709 |
} |
2709 |
} |
| 2710 |
} |
2710 |
} |
| 2711 |
|
2711 |
|
| 2712 |
// Insert the other groups (one at a time). |
2712 |
// Insert the other groups (one at a time). |
| 2713 |
for (auto &BG : BitGroups) { |
2713 |
for (auto &BG : BitGroups) { |
| 2714 |
if (!Res) |
2714 |
if (!Res) |
| 2715 |
Res = SelectRotMask64(BG.V, dl, BG.RLAmt, BG.Repl32, BG.StartIdx, |
2715 |
Res = SelectRotMask64(BG.V, dl, BG.RLAmt, BG.Repl32, BG.StartIdx, |
| 2716 |
BG.EndIdx, InstCnt); |
2716 |
BG.EndIdx, InstCnt); |
| 2717 |
else |
2717 |
else |
| 2718 |
Res = SelectRotMaskIns64(Res, BG.V, dl, BG.RLAmt, BG.Repl32, |
2718 |
Res = SelectRotMaskIns64(Res, BG.V, dl, BG.RLAmt, BG.Repl32, |
| 2719 |
BG.StartIdx, BG.EndIdx, InstCnt); |
2719 |
BG.StartIdx, BG.EndIdx, InstCnt); |
| 2720 |
} |
2720 |
} |
| 2721 |
|
2721 |
|
| 2722 |
if (LateMask) { |
2722 |
if (LateMask) { |
| 2723 |
uint64_t Mask = getZerosMask(); |
2723 |
uint64_t Mask = getZerosMask(); |
| 2724 |
|
2724 |
|
| 2725 |
// We can use the 32-bit andi/andis technique if the mask does not |
2725 |
// We can use the 32-bit andi/andis technique if the mask does not |
| 2726 |
// require any higher-order bits. This can save an instruction compared |
2726 |
// require any higher-order bits. This can save an instruction compared |
| 2727 |
// to always using the general 64-bit technique. |
2727 |
// to always using the general 64-bit technique. |
| 2728 |
bool Use32BitInsts = isUInt<32>(Mask); |
2728 |
bool Use32BitInsts = isUInt<32>(Mask); |
| 2729 |
// Compute the masks for andi/andis that would be necessary. |
2729 |
// Compute the masks for andi/andis that would be necessary. |
| 2730 |
unsigned ANDIMask = (Mask & UINT16_MAX), |
2730 |
unsigned ANDIMask = (Mask & UINT16_MAX), |
| 2731 |
ANDISMask = (Mask >> 16) & UINT16_MAX; |
2731 |
ANDISMask = (Mask >> 16) & UINT16_MAX; |
| 2732 |
|
2732 |
|
| 2733 |
if (Use32BitInsts) { |
2733 |
if (Use32BitInsts) { |
| 2734 |
assert((ANDIMask != 0 || ANDISMask != 0) && |
2734 |
assert((ANDIMask != 0 || ANDISMask != 0) && |
| 2735 |
"No set bits in mask when using 32-bit ands for 64-bit value"); |
2735 |
"No set bits in mask when using 32-bit ands for 64-bit value"); |
| 2736 |
|
2736 |
|
| 2737 |
if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) + |
2737 |
if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) + |
| 2738 |
(unsigned) (ANDISMask != 0) + |
2738 |
(unsigned) (ANDISMask != 0) + |
| 2739 |
(unsigned) (ANDIMask != 0 && ANDISMask != 0); |
2739 |
(unsigned) (ANDIMask != 0 && ANDISMask != 0); |
| 2740 |
|
2740 |
|
| 2741 |
SDValue ANDIVal, ANDISVal; |
2741 |
SDValue ANDIVal, ANDISVal; |
| 2742 |
if (ANDIMask != 0) |
2742 |
if (ANDIMask != 0) |
| 2743 |
ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64, |
2743 |
ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64, |
| 2744 |
ExtendToInt64(Res, dl), |
2744 |
ExtendToInt64(Res, dl), |
| 2745 |
getI32Imm(ANDIMask, dl)), |
2745 |
getI32Imm(ANDIMask, dl)), |
| 2746 |
0); |
2746 |
0); |
| 2747 |
if (ANDISMask != 0) |
2747 |
if (ANDISMask != 0) |
| 2748 |
ANDISVal = |
2748 |
ANDISVal = |
| 2749 |
SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64, |
2749 |
SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64, |
| 2750 |
ExtendToInt64(Res, dl), |
2750 |
ExtendToInt64(Res, dl), |
| 2751 |
getI32Imm(ANDISMask, dl)), |
2751 |
getI32Imm(ANDISMask, dl)), |
| 2752 |
0); |
2752 |
0); |
| 2753 |
|
2753 |
|
| 2754 |
if (!ANDIVal) |
2754 |
if (!ANDIVal) |
| 2755 |
Res = ANDISVal; |
2755 |
Res = ANDISVal; |
| 2756 |
else if (!ANDISVal) |
2756 |
else if (!ANDISVal) |
| 2757 |
Res = ANDIVal; |
2757 |
Res = ANDIVal; |
| 2758 |
else |
2758 |
else |
| 2759 |
Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, |
2759 |
Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, |
| 2760 |
ExtendToInt64(ANDIVal, dl), ANDISVal), 0); |
2760 |
ExtendToInt64(ANDIVal, dl), ANDISVal), 0); |
| 2761 |
} else { |
2761 |
} else { |
| 2762 |
unsigned NumOfSelectInsts = 0; |
2762 |
unsigned NumOfSelectInsts = 0; |
| 2763 |
SDValue MaskVal = |
2763 |
SDValue MaskVal = |
| 2764 |
SDValue(selectI64Imm(CurDAG, dl, Mask, &NumOfSelectInsts), 0); |
2764 |
SDValue(selectI64Imm(CurDAG, dl, Mask, &NumOfSelectInsts), 0); |
| 2765 |
Res = SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64, |
2765 |
Res = SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64, |
| 2766 |
ExtendToInt64(Res, dl), MaskVal), |
2766 |
ExtendToInt64(Res, dl), MaskVal), |
| 2767 |
0); |
2767 |
0); |
| 2768 |
if (InstCnt) |
2768 |
if (InstCnt) |
| 2769 |
*InstCnt += NumOfSelectInsts + /* and */ 1; |
2769 |
*InstCnt += NumOfSelectInsts + /* and */ 1; |
| 2770 |
} |
2770 |
} |
| 2771 |
} |
2771 |
} |
| 2772 |
|
2772 |
|
| 2773 |
return Res.getNode(); |
2773 |
return Res.getNode(); |
| 2774 |
} |
2774 |
} |
| 2775 |
|
2775 |
|
| 2776 |
SDNode *Select(SDNode *N, bool LateMask, unsigned *InstCnt = nullptr) { |
2776 |
SDNode *Select(SDNode *N, bool LateMask, unsigned *InstCnt = nullptr) { |
| 2777 |
// Fill in BitGroups. |
2777 |
// Fill in BitGroups. |
| 2778 |
collectBitGroups(LateMask); |
2778 |
collectBitGroups(LateMask); |
| 2779 |
if (BitGroups.empty()) |
2779 |
if (BitGroups.empty()) |
| 2780 |
return nullptr; |
2780 |
return nullptr; |
| 2781 |
|
2781 |
|
| 2782 |
// For 64-bit values, figure out when we can use 32-bit instructions. |
2782 |
// For 64-bit values, figure out when we can use 32-bit instructions. |
| 2783 |
if (Bits.size() == 64) |
2783 |
if (Bits.size() == 64) |
| 2784 |
assignRepl32BitGroups(); |
2784 |
assignRepl32BitGroups(); |
| 2785 |
|
2785 |
|
| 2786 |
// Fill in ValueRotsVec. |
2786 |
// Fill in ValueRotsVec. |
| 2787 |
collectValueRotInfo(); |
2787 |
collectValueRotInfo(); |
| 2788 |
|
2788 |
|
| 2789 |
if (Bits.size() == 32) { |
2789 |
if (Bits.size() == 32) { |
| 2790 |
return Select32(N, LateMask, InstCnt); |
2790 |
return Select32(N, LateMask, InstCnt); |
| 2791 |
} else { |
2791 |
} else { |
| 2792 |
assert(Bits.size() == 64 && "Not 64 bits here?"); |
2792 |
assert(Bits.size() == 64 && "Not 64 bits here?"); |
| 2793 |
return Select64(N, LateMask, InstCnt); |
2793 |
return Select64(N, LateMask, InstCnt); |
| 2794 |
} |
2794 |
} |
| 2795 |
|
2795 |
|
| 2796 |
return nullptr; |
2796 |
return nullptr; |
| 2797 |
} |
2797 |
} |
| 2798 |
|
2798 |
|
| 2799 |
void eraseMatchingBitGroups(function_ref F) { |
2799 |
void eraseMatchingBitGroups(function_ref F) { |
| 2800 |
erase_if(BitGroups, F); |
2800 |
erase_if(BitGroups, F); |
| 2801 |
} |
2801 |
} |
| 2802 |
|
2802 |
|
| 2803 |
SmallVector Bits; |
2803 |
SmallVector Bits; |
| 2804 |
|
2804 |
|
| 2805 |
bool NeedMask = false; |
2805 |
bool NeedMask = false; |
| 2806 |
SmallVector RLAmt; |
2806 |
SmallVector RLAmt; |
| 2807 |
|
2807 |
|
| 2808 |
SmallVector BitGroups; |
2808 |
SmallVector BitGroups; |
| 2809 |
|
2809 |
|
| 2810 |
DenseMap, ValueRotInfo> ValueRots; |
2810 |
DenseMap, ValueRotInfo> ValueRots; |
| 2811 |
SmallVector ValueRotsVec; |
2811 |
SmallVector ValueRotsVec; |
| 2812 |
|
2812 |
|
| 2813 |
SelectionDAG *CurDAG = nullptr; |
2813 |
SelectionDAG *CurDAG = nullptr; |
| 2814 |
|
2814 |
|
| 2815 |
public: |
2815 |
public: |
| 2816 |
BitPermutationSelector(SelectionDAG *DAG) |
2816 |
BitPermutationSelector(SelectionDAG *DAG) |
| 2817 |
: CurDAG(DAG) {} |
2817 |
: CurDAG(DAG) {} |
| 2818 |
|
2818 |
|
| 2819 |
// Here we try to match complex bit permutations into a set of |
2819 |
// Here we try to match complex bit permutations into a set of |
| 2820 |
// rotate-and-shift/shift/and/or instructions, using a set of heuristics |
2820 |
// rotate-and-shift/shift/and/or instructions, using a set of heuristics |
| 2821 |
// known to produce optimal code for common cases (like i32 byte swapping). |
2821 |
// known to produce optimal code for common cases (like i32 byte swapping). |
| 2822 |
SDNode *Select(SDNode *N) { |
2822 |
SDNode *Select(SDNode *N) { |
| 2823 |
Memoizer.clear(); |
2823 |
Memoizer.clear(); |
| 2824 |
auto Result = |
2824 |
auto Result = |
| 2825 |
getValueBits(SDValue(N, 0), N->getValueType(0).getSizeInBits()); |
2825 |
getValueBits(SDValue(N, 0), N->getValueType(0).getSizeInBits()); |
| 2826 |
if (!Result.first) |
2826 |
if (!Result.first) |
| 2827 |
return nullptr; |
2827 |
return nullptr; |
| 2828 |
Bits = std::move(*Result.second); |
2828 |
Bits = std::move(*Result.second); |
| 2829 |
|
2829 |
|
| 2830 |
LLVM_DEBUG(dbgs() << "Considering bit-permutation-based instruction" |
2830 |
LLVM_DEBUG(dbgs() << "Considering bit-permutation-based instruction" |
| 2831 |
" selection for: "); |
2831 |
" selection for: "); |
| 2832 |
LLVM_DEBUG(N->dump(CurDAG)); |
2832 |
LLVM_DEBUG(N->dump(CurDAG)); |
| 2833 |
|
2833 |
|
| 2834 |
// Fill it RLAmt and set NeedMask. |
2834 |
// Fill it RLAmt and set NeedMask. |
| 2835 |
computeRotationAmounts(); |
2835 |
computeRotationAmounts(); |
| 2836 |
|
2836 |
|
| 2837 |
if (!NeedMask) |
2837 |
if (!NeedMask) |
| 2838 |
return Select(N, false); |
2838 |
return Select(N, false); |
| 2839 |
|
2839 |
|
| 2840 |
// We currently have two techniques for handling results with zeros: early |
2840 |
// We currently have two techniques for handling results with zeros: early |
| 2841 |
// masking (the default) and late masking. Late masking is sometimes more |
2841 |
// masking (the default) and late masking. Late masking is sometimes more |
| 2842 |
// efficient, but because the structure of the bit groups is different, it |
2842 |
// efficient, but because the structure of the bit groups is different, it |
| 2843 |
// is hard to tell without generating both and comparing the results. With |
2843 |
// is hard to tell without generating both and comparing the results. With |
| 2844 |
// late masking, we ignore zeros in the resulting value when inserting each |
2844 |
// late masking, we ignore zeros in the resulting value when inserting each |
| 2845 |
// set of bit groups, and then mask in the zeros at the end. With early |
2845 |
// set of bit groups, and then mask in the zeros at the end. With early |
| 2846 |
// masking, we only insert the non-zero parts of the result at every step. |
2846 |
// masking, we only insert the non-zero parts of the result at every step. |
| 2847 |
|
2847 |
|
| 2848 |
unsigned InstCnt = 0, InstCntLateMask = 0; |
2848 |
unsigned InstCnt = 0, InstCntLateMask = 0; |
| 2849 |
LLVM_DEBUG(dbgs() << "\tEarly masking:\n"); |
2849 |
LLVM_DEBUG(dbgs() << "\tEarly masking:\n"); |
| 2850 |
SDNode *RN = Select(N, false, &InstCnt); |
2850 |
SDNode *RN = Select(N, false, &InstCnt); |
| 2851 |
LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCnt << " instructions\n"); |
2851 |
LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCnt << " instructions\n"); |
| 2852 |
|
2852 |
|
| 2853 |
LLVM_DEBUG(dbgs() << "\tLate masking:\n"); |
2853 |
LLVM_DEBUG(dbgs() << "\tLate masking:\n"); |
| 2854 |
SDNode *RNLM = Select(N, true, &InstCntLateMask); |
2854 |
SDNode *RNLM = Select(N, true, &InstCntLateMask); |
| 2855 |
LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCntLateMask |
2855 |
LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCntLateMask |
| 2856 |
<< " instructions\n"); |
2856 |
<< " instructions\n"); |
| 2857 |
|
2857 |
|
| 2858 |
if (InstCnt <= InstCntLateMask) { |
2858 |
if (InstCnt <= InstCntLateMask) { |
| 2859 |
LLVM_DEBUG(dbgs() << "\tUsing early-masking for isel\n"); |
2859 |
LLVM_DEBUG(dbgs() << "\tUsing early-masking for isel\n"); |
| 2860 |
return RN; |
2860 |
return RN; |
| 2861 |
} |
2861 |
} |
| 2862 |
|
2862 |
|
| 2863 |
LLVM_DEBUG(dbgs() << "\tUsing late-masking for isel\n"); |
2863 |
LLVM_DEBUG(dbgs() << "\tUsing late-masking for isel\n"); |
| 2864 |
return RNLM; |
2864 |
return RNLM; |
| 2865 |
} |
2865 |
} |
| 2866 |
}; |
2866 |
}; |
| 2867 |
|
2867 |
|
| 2868 |
class IntegerCompareEliminator { |
2868 |
class IntegerCompareEliminator { |
| 2869 |
SelectionDAG *CurDAG; |
2869 |
SelectionDAG *CurDAG; |
| 2870 |
PPCDAGToDAGISel *S; |
2870 |
PPCDAGToDAGISel *S; |
| 2871 |
// Conversion type for interpreting results of a 32-bit instruction as |
2871 |
// Conversion type for interpreting results of a 32-bit instruction as |
| 2872 |
// a 64-bit value or vice versa. |
2872 |
// a 64-bit value or vice versa. |
| 2873 |
enum ExtOrTruncConversion { Ext, Trunc }; |
2873 |
enum ExtOrTruncConversion { Ext, Trunc }; |
| 2874 |
|
2874 |
|
| 2875 |
// Modifiers to guide how an ISD::SETCC node's result is to be computed |
2875 |
// Modifiers to guide how an ISD::SETCC node's result is to be computed |
| 2876 |
// in a GPR. |
2876 |
// in a GPR. |
| 2877 |
// ZExtOrig - use the original condition code, zero-extend value |
2877 |
// ZExtOrig - use the original condition code, zero-extend value |
| 2878 |
// ZExtInvert - invert the condition code, zero-extend value |
2878 |
// ZExtInvert - invert the condition code, zero-extend value |
| 2879 |
// SExtOrig - use the original condition code, sign-extend value |
2879 |
// SExtOrig - use the original condition code, sign-extend value |
| 2880 |
// SExtInvert - invert the condition code, sign-extend value |
2880 |
// SExtInvert - invert the condition code, sign-extend value |
| 2881 |
enum SetccInGPROpts { ZExtOrig, ZExtInvert, SExtOrig, SExtInvert }; |
2881 |
enum SetccInGPROpts { ZExtOrig, ZExtInvert, SExtOrig, SExtInvert }; |
| 2882 |
|
2882 |
|
| 2883 |
// Comparisons against zero to emit GPR code sequences for. Each of these |
2883 |
// Comparisons against zero to emit GPR code sequences for. Each of these |
| 2884 |
// sequences may need to be emitted for two or more equivalent patterns. |
2884 |
// sequences may need to be emitted for two or more equivalent patterns. |
| 2885 |
// For example (a >= 0) == (a > -1). The direction of the comparison () |
2885 |
// For example (a >= 0) == (a > -1). The direction of the comparison () |
| 2886 |
// matters as well as the extension type: sext (-1/0), zext (1/0). |
2886 |
// matters as well as the extension type: sext (-1/0), zext (1/0). |
| 2887 |
// GEZExt - (zext (LHS >= 0)) |
2887 |
// GEZExt - (zext (LHS >= 0)) |
| 2888 |
// GESExt - (sext (LHS >= 0)) |
2888 |
// GESExt - (sext (LHS >= 0)) |
| 2889 |
// LEZExt - (zext (LHS <= 0)) |
2889 |
// LEZExt - (zext (LHS <= 0)) |
| 2890 |
// LESExt - (sext (LHS <= 0)) |
2890 |
// LESExt - (sext (LHS <= 0)) |
| 2891 |
enum ZeroCompare { GEZExt, GESExt, LEZExt, LESExt }; |
2891 |
enum ZeroCompare { GEZExt, GESExt, LEZExt, LESExt }; |
| 2892 |
|
2892 |
|
| 2893 |
SDNode *tryEXTEND(SDNode *N); |
2893 |
SDNode *tryEXTEND(SDNode *N); |
| 2894 |
SDNode *tryLogicOpOfCompares(SDNode *N); |
2894 |
SDNode *tryLogicOpOfCompares(SDNode *N); |
| 2895 |
SDValue computeLogicOpInGPR(SDValue LogicOp); |
2895 |
SDValue computeLogicOpInGPR(SDValue LogicOp); |
| 2896 |
SDValue signExtendInputIfNeeded(SDValue Input); |
2896 |
SDValue signExtendInputIfNeeded(SDValue Input); |
| 2897 |
SDValue zeroExtendInputIfNeeded(SDValue Input); |
2897 |
SDValue zeroExtendInputIfNeeded(SDValue Input); |
| 2898 |
SDValue addExtOrTrunc(SDValue NatWidthRes, ExtOrTruncConversion Conv); |
2898 |
SDValue addExtOrTrunc(SDValue NatWidthRes, ExtOrTruncConversion Conv); |
| 2899 |
SDValue getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl, |
2899 |
SDValue getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl, |
| 2900 |
ZeroCompare CmpTy); |
2900 |
ZeroCompare CmpTy); |
| 2901 |
SDValue get32BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
2901 |
SDValue get32BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
| 2902 |
int64_t RHSValue, SDLoc dl); |
2902 |
int64_t RHSValue, SDLoc dl); |
| 2903 |
SDValue get32BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
2903 |
SDValue get32BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
| 2904 |
int64_t RHSValue, SDLoc dl); |
2904 |
int64_t RHSValue, SDLoc dl); |
| 2905 |
SDValue get64BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
2905 |
SDValue get64BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
| 2906 |
int64_t RHSValue, SDLoc dl); |
2906 |
int64_t RHSValue, SDLoc dl); |
| 2907 |
SDValue get64BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
2907 |
SDValue get64BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
| 2908 |
int64_t RHSValue, SDLoc dl); |
2908 |
int64_t RHSValue, SDLoc dl); |
| 2909 |
SDValue getSETCCInGPR(SDValue Compare, SetccInGPROpts ConvOpts); |
2909 |
SDValue getSETCCInGPR(SDValue Compare, SetccInGPROpts ConvOpts); |
| 2910 |
|
2910 |
|
| 2911 |
public: |
2911 |
public: |
| 2912 |
IntegerCompareEliminator(SelectionDAG *DAG, |
2912 |
IntegerCompareEliminator(SelectionDAG *DAG, |
| 2913 |
PPCDAGToDAGISel *Sel) : CurDAG(DAG), S(Sel) { |
2913 |
PPCDAGToDAGISel *Sel) : CurDAG(DAG), S(Sel) { |
| 2914 |
assert(CurDAG->getTargetLoweringInfo() |
2914 |
assert(CurDAG->getTargetLoweringInfo() |
| 2915 |
.getPointerTy(CurDAG->getDataLayout()).getSizeInBits() == 64 && |
2915 |
.getPointerTy(CurDAG->getDataLayout()).getSizeInBits() == 64 && |
| 2916 |
"Only expecting to use this on 64 bit targets."); |
2916 |
"Only expecting to use this on 64 bit targets."); |
| 2917 |
} |
2917 |
} |
| 2918 |
SDNode *Select(SDNode *N) { |
2918 |
SDNode *Select(SDNode *N) { |
| 2919 |
if (CmpInGPR == ICGPR_None) |
2919 |
if (CmpInGPR == ICGPR_None) |
| 2920 |
return nullptr; |
2920 |
return nullptr; |
| 2921 |
switch (N->getOpcode()) { |
2921 |
switch (N->getOpcode()) { |
| 2922 |
default: break; |
2922 |
default: break; |
| 2923 |
case ISD::ZERO_EXTEND: |
2923 |
case ISD::ZERO_EXTEND: |
| 2924 |
if (CmpInGPR == ICGPR_Sext || CmpInGPR == ICGPR_SextI32 || |
2924 |
if (CmpInGPR == ICGPR_Sext || CmpInGPR == ICGPR_SextI32 || |
| 2925 |
CmpInGPR == ICGPR_SextI64) |
2925 |
CmpInGPR == ICGPR_SextI64) |
| 2926 |
return nullptr; |
2926 |
return nullptr; |
| 2927 |
[[fallthrough]]; |
2927 |
[[fallthrough]]; |
| 2928 |
case ISD::SIGN_EXTEND: |
2928 |
case ISD::SIGN_EXTEND: |
| 2929 |
if (CmpInGPR == ICGPR_Zext || CmpInGPR == ICGPR_ZextI32 || |
2929 |
if (CmpInGPR == ICGPR_Zext || CmpInGPR == ICGPR_ZextI32 || |
| 2930 |
CmpInGPR == ICGPR_ZextI64) |
2930 |
CmpInGPR == ICGPR_ZextI64) |
| 2931 |
return nullptr; |
2931 |
return nullptr; |
| 2932 |
return tryEXTEND(N); |
2932 |
return tryEXTEND(N); |
| 2933 |
case ISD::AND: |
2933 |
case ISD::AND: |
| 2934 |
case ISD::OR: |
2934 |
case ISD::OR: |
| 2935 |
case ISD::XOR: |
2935 |
case ISD::XOR: |
| 2936 |
return tryLogicOpOfCompares(N); |
2936 |
return tryLogicOpOfCompares(N); |
| 2937 |
} |
2937 |
} |
| 2938 |
return nullptr; |
2938 |
return nullptr; |
| 2939 |
} |
2939 |
} |
| 2940 |
}; |
2940 |
}; |
| 2941 |
|
2941 |
|
| 2942 |
// The obvious case for wanting to keep the value in a GPR. Namely, the |
2942 |
// The obvious case for wanting to keep the value in a GPR. Namely, the |
| 2943 |
// result of the comparison is actually needed in a GPR. |
2943 |
// result of the comparison is actually needed in a GPR. |
| 2944 |
SDNode *IntegerCompareEliminator::tryEXTEND(SDNode *N) { |
2944 |
SDNode *IntegerCompareEliminator::tryEXTEND(SDNode *N) { |
| 2945 |
assert((N->getOpcode() == ISD::ZERO_EXTEND || |
2945 |
assert((N->getOpcode() == ISD::ZERO_EXTEND || |
| 2946 |
N->getOpcode() == ISD::SIGN_EXTEND) && |
2946 |
N->getOpcode() == ISD::SIGN_EXTEND) && |
| 2947 |
"Expecting a zero/sign extend node!"); |
2947 |
"Expecting a zero/sign extend node!"); |
| 2948 |
SDValue WideRes; |
2948 |
SDValue WideRes; |
| 2949 |
// If we are zero-extending the result of a logical operation on i1 |
2949 |
// If we are zero-extending the result of a logical operation on i1 |
| 2950 |
// values, we can keep the values in GPRs. |
2950 |
// values, we can keep the values in GPRs. |
| 2951 |
if (ISD::isBitwiseLogicOp(N->getOperand(0).getOpcode()) && |
2951 |
if (ISD::isBitwiseLogicOp(N->getOperand(0).getOpcode()) && |
| 2952 |
N->getOperand(0).getValueType() == MVT::i1 && |
2952 |
N->getOperand(0).getValueType() == MVT::i1 && |
| 2953 |
N->getOpcode() == ISD::ZERO_EXTEND) |
2953 |
N->getOpcode() == ISD::ZERO_EXTEND) |
| 2954 |
WideRes = computeLogicOpInGPR(N->getOperand(0)); |
2954 |
WideRes = computeLogicOpInGPR(N->getOperand(0)); |
| 2955 |
else if (N->getOperand(0).getOpcode() != ISD::SETCC) |
2955 |
else if (N->getOperand(0).getOpcode() != ISD::SETCC) |
| 2956 |
return nullptr; |
2956 |
return nullptr; |
| 2957 |
else |
2957 |
else |
| 2958 |
WideRes = |
2958 |
WideRes = |
| 2959 |
getSETCCInGPR(N->getOperand(0), |
2959 |
getSETCCInGPR(N->getOperand(0), |
| 2960 |
N->getOpcode() == ISD::SIGN_EXTEND ? |
2960 |
N->getOpcode() == ISD::SIGN_EXTEND ? |
| 2961 |
SetccInGPROpts::SExtOrig : SetccInGPROpts::ZExtOrig); |
2961 |
SetccInGPROpts::SExtOrig : SetccInGPROpts::ZExtOrig); |
| 2962 |
|
2962 |
|
| 2963 |
if (!WideRes) |
2963 |
if (!WideRes) |
| 2964 |
return nullptr; |
2964 |
return nullptr; |
| 2965 |
|
2965 |
|
| 2966 |
SDLoc dl(N); |
2966 |
SDLoc dl(N); |
| 2967 |
bool Input32Bit = WideRes.getValueType() == MVT::i32; |
2967 |
bool Input32Bit = WideRes.getValueType() == MVT::i32; |
| 2968 |
bool Output32Bit = N->getValueType(0) == MVT::i32; |
2968 |
bool Output32Bit = N->getValueType(0) == MVT::i32; |
| 2969 |
|
2969 |
|
| 2970 |
NumSextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 1 : 0; |
2970 |
NumSextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 1 : 0; |
| 2971 |
NumZextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 0 : 1; |
2971 |
NumZextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 0 : 1; |
| 2972 |
|
2972 |
|
| 2973 |
SDValue ConvOp = WideRes; |
2973 |
SDValue ConvOp = WideRes; |
| 2974 |
if (Input32Bit != Output32Bit) |
2974 |
if (Input32Bit != Output32Bit) |
| 2975 |
ConvOp = addExtOrTrunc(WideRes, Input32Bit ? ExtOrTruncConversion::Ext : |
2975 |
ConvOp = addExtOrTrunc(WideRes, Input32Bit ? ExtOrTruncConversion::Ext : |
| 2976 |
ExtOrTruncConversion::Trunc); |
2976 |
ExtOrTruncConversion::Trunc); |
| 2977 |
return ConvOp.getNode(); |
2977 |
return ConvOp.getNode(); |
| 2978 |
} |
2978 |
} |
| 2979 |
|
2979 |
|
| 2980 |
// Attempt to perform logical operations on the results of comparisons while |
2980 |
// Attempt to perform logical operations on the results of comparisons while |
| 2981 |
// keeping the values in GPRs. Without doing so, these would end up being |
2981 |
// keeping the values in GPRs. Without doing so, these would end up being |
| 2982 |
// lowered to CR-logical operations which suffer from significant latency and |
2982 |
// lowered to CR-logical operations which suffer from significant latency and |
| 2983 |
// low ILP. |
2983 |
// low ILP. |
| 2984 |
SDNode *IntegerCompareEliminator::tryLogicOpOfCompares(SDNode *N) { |
2984 |
SDNode *IntegerCompareEliminator::tryLogicOpOfCompares(SDNode *N) { |
| 2985 |
if (N->getValueType(0) != MVT::i1) |
2985 |
if (N->getValueType(0) != MVT::i1) |
| 2986 |
return nullptr; |
2986 |
return nullptr; |
| 2987 |
assert(ISD::isBitwiseLogicOp(N->getOpcode()) && |
2987 |
assert(ISD::isBitwiseLogicOp(N->getOpcode()) && |
| 2988 |
"Expected a logic operation on setcc results."); |
2988 |
"Expected a logic operation on setcc results."); |
| 2989 |
SDValue LoweredLogical = computeLogicOpInGPR(SDValue(N, 0)); |
2989 |
SDValue LoweredLogical = computeLogicOpInGPR(SDValue(N, 0)); |
| 2990 |
if (!LoweredLogical) |
2990 |
if (!LoweredLogical) |
| 2991 |
return nullptr; |
2991 |
return nullptr; |
| 2992 |
|
2992 |
|
| 2993 |
SDLoc dl(N); |
2993 |
SDLoc dl(N); |
| 2994 |
bool IsBitwiseNegate = LoweredLogical.getMachineOpcode() == PPC::XORI8; |
2994 |
bool IsBitwiseNegate = LoweredLogical.getMachineOpcode() == PPC::XORI8; |
| 2995 |
unsigned SubRegToExtract = IsBitwiseNegate ? PPC::sub_eq : PPC::sub_gt; |
2995 |
unsigned SubRegToExtract = IsBitwiseNegate ? PPC::sub_eq : PPC::sub_gt; |
| 2996 |
SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32); |
2996 |
SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32); |
| 2997 |
SDValue LHS = LoweredLogical.getOperand(0); |
2997 |
SDValue LHS = LoweredLogical.getOperand(0); |
| 2998 |
SDValue RHS = LoweredLogical.getOperand(1); |
2998 |
SDValue RHS = LoweredLogical.getOperand(1); |
| 2999 |
SDValue WideOp; |
2999 |
SDValue WideOp; |
| 3000 |
SDValue OpToConvToRecForm; |
3000 |
SDValue OpToConvToRecForm; |
| 3001 |
|
3001 |
|
| 3002 |
// Look through any 32-bit to 64-bit implicit extend nodes to find the |
3002 |
// Look through any 32-bit to 64-bit implicit extend nodes to find the |
| 3003 |
// opcode that is input to the XORI. |
3003 |
// opcode that is input to the XORI. |
| 3004 |
if (IsBitwiseNegate && |
3004 |
if (IsBitwiseNegate && |
| 3005 |
LoweredLogical.getOperand(0).getMachineOpcode() == PPC::INSERT_SUBREG) |
3005 |
LoweredLogical.getOperand(0).getMachineOpcode() == PPC::INSERT_SUBREG) |
| 3006 |
OpToConvToRecForm = LoweredLogical.getOperand(0).getOperand(1); |
3006 |
OpToConvToRecForm = LoweredLogical.getOperand(0).getOperand(1); |
| 3007 |
else if (IsBitwiseNegate) |
3007 |
else if (IsBitwiseNegate) |
| 3008 |
// If the input to the XORI isn't an extension, that's what we're after. |
3008 |
// If the input to the XORI isn't an extension, that's what we're after. |
| 3009 |
OpToConvToRecForm = LoweredLogical.getOperand(0); |
3009 |
OpToConvToRecForm = LoweredLogical.getOperand(0); |
| 3010 |
else |
3010 |
else |
| 3011 |
// If this is not an XORI, it is a reg-reg logical op and we can convert |
3011 |
// If this is not an XORI, it is a reg-reg logical op and we can convert |
| 3012 |
// it to record-form. |
3012 |
// it to record-form. |
| 3013 |
OpToConvToRecForm = LoweredLogical; |
3013 |
OpToConvToRecForm = LoweredLogical; |
| 3014 |
|
3014 |
|
| 3015 |
// Get the record-form version of the node we're looking to use to get the |
3015 |
// Get the record-form version of the node we're looking to use to get the |
| 3016 |
// CR result from. |
3016 |
// CR result from. |
| 3017 |
uint16_t NonRecOpc = OpToConvToRecForm.getMachineOpcode(); |
3017 |
uint16_t NonRecOpc = OpToConvToRecForm.getMachineOpcode(); |
| 3018 |
int NewOpc = PPCInstrInfo::getRecordFormOpcode(NonRecOpc); |
3018 |
int NewOpc = PPCInstrInfo::getRecordFormOpcode(NonRecOpc); |
| 3019 |
|
3019 |
|
| 3020 |
// Convert the right node to record-form. This is either the logical we're |
3020 |
// Convert the right node to record-form. This is either the logical we're |
| 3021 |
// looking at or it is the input node to the negation (if we're looking at |
3021 |
// looking at or it is the input node to the negation (if we're looking at |
| 3022 |
// a bitwise negation). |
3022 |
// a bitwise negation). |
| 3023 |
if (NewOpc != -1 && IsBitwiseNegate) { |
3023 |
if (NewOpc != -1 && IsBitwiseNegate) { |
| 3024 |
// The input to the XORI has a record-form. Use it. |
3024 |
// The input to the XORI has a record-form. Use it. |
| 3025 |
assert(LoweredLogical.getConstantOperandVal(1) == 1 && |
3025 |
assert(LoweredLogical.getConstantOperandVal(1) == 1 && |
| 3026 |
"Expected a PPC::XORI8 only for bitwise negation."); |
3026 |
"Expected a PPC::XORI8 only for bitwise negation."); |
| 3027 |
// Emit the record-form instruction. |
3027 |
// Emit the record-form instruction. |
| 3028 |
std::vector Ops; |
3028 |
std::vector Ops; |
| 3029 |
for (int i = 0, e = OpToConvToRecForm.getNumOperands(); i < e; i++) |
3029 |
for (int i = 0, e = OpToConvToRecForm.getNumOperands(); i < e; i++) |
| 3030 |
Ops.push_back(OpToConvToRecForm.getOperand(i)); |
3030 |
Ops.push_back(OpToConvToRecForm.getOperand(i)); |
| 3031 |
|
3031 |
|
| 3032 |
WideOp = |
3032 |
WideOp = |
| 3033 |
SDValue(CurDAG->getMachineNode(NewOpc, dl, |
3033 |
SDValue(CurDAG->getMachineNode(NewOpc, dl, |
| 3034 |
OpToConvToRecForm.getValueType(), |
3034 |
OpToConvToRecForm.getValueType(), |
| 3035 |
MVT::Glue, Ops), 0); |
3035 |
MVT::Glue, Ops), 0); |
| 3036 |
} else { |
3036 |
} else { |
| 3037 |
assert((NewOpc != -1 || !IsBitwiseNegate) && |
3037 |
assert((NewOpc != -1 || !IsBitwiseNegate) && |
| 3038 |
"No record form available for AND8/OR8/XOR8?"); |
3038 |
"No record form available for AND8/OR8/XOR8?"); |
| 3039 |
WideOp = |
3039 |
WideOp = |
| 3040 |
SDValue(CurDAG->getMachineNode(NewOpc == -1 ? PPC::ANDI8_rec : NewOpc, |
3040 |
SDValue(CurDAG->getMachineNode(NewOpc == -1 ? PPC::ANDI8_rec : NewOpc, |
| 3041 |
dl, MVT::i64, MVT::Glue, LHS, RHS), |
3041 |
dl, MVT::i64, MVT::Glue, LHS, RHS), |
| 3042 |
0); |
3042 |
0); |
| 3043 |
} |
3043 |
} |
| 3044 |
|
3044 |
|
| 3045 |
// Select this node to a single bit from CR0 set by the record-form node |
3045 |
// Select this node to a single bit from CR0 set by the record-form node |
| 3046 |
// just created. For bitwise negation, use the EQ bit which is the equivalent |
3046 |
// just created. For bitwise negation, use the EQ bit which is the equivalent |
| 3047 |
// of negating the result (i.e. it is a bit set when the result of the |
3047 |
// of negating the result (i.e. it is a bit set when the result of the |
| 3048 |
// operation is zero). |
3048 |
// operation is zero). |
| 3049 |
SDValue SRIdxVal = |
3049 |
SDValue SRIdxVal = |
| 3050 |
CurDAG->getTargetConstant(SubRegToExtract, dl, MVT::i32); |
3050 |
CurDAG->getTargetConstant(SubRegToExtract, dl, MVT::i32); |
| 3051 |
SDValue CRBit = |
3051 |
SDValue CRBit = |
| 3052 |
SDValue(CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, |
3052 |
SDValue(CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, |
| 3053 |
MVT::i1, CR0Reg, SRIdxVal, |
3053 |
MVT::i1, CR0Reg, SRIdxVal, |
| 3054 |
WideOp.getValue(1)), 0); |
3054 |
WideOp.getValue(1)), 0); |
| 3055 |
return CRBit.getNode(); |
3055 |
return CRBit.getNode(); |
| 3056 |
} |
3056 |
} |
| 3057 |
|
3057 |
|
| 3058 |
// Lower a logical operation on i1 values into a GPR sequence if possible. |
3058 |
// Lower a logical operation on i1 values into a GPR sequence if possible. |
| 3059 |
// The result can be kept in a GPR if requested. |
3059 |
// The result can be kept in a GPR if requested. |
| 3060 |
// Three types of inputs can be handled: |
3060 |
// Three types of inputs can be handled: |
| 3061 |
// - SETCC |
3061 |
// - SETCC |
| 3062 |
// - TRUNCATE |
3062 |
// - TRUNCATE |
| 3063 |
// - Logical operation (AND/OR/XOR) |
3063 |
// - Logical operation (AND/OR/XOR) |
| 3064 |
// There is also a special case that is handled (namely a complement operation |
3064 |
// There is also a special case that is handled (namely a complement operation |
| 3065 |
// achieved with xor %a, -1). |
3065 |
// achieved with xor %a, -1). |
| 3066 |
SDValue IntegerCompareEliminator::computeLogicOpInGPR(SDValue LogicOp) { |
3066 |
SDValue IntegerCompareEliminator::computeLogicOpInGPR(SDValue LogicOp) { |
| 3067 |
assert(ISD::isBitwiseLogicOp(LogicOp.getOpcode()) && |
3067 |
assert(ISD::isBitwiseLogicOp(LogicOp.getOpcode()) && |
| 3068 |
"Can only handle logic operations here."); |
3068 |
"Can only handle logic operations here."); |
| 3069 |
assert(LogicOp.getValueType() == MVT::i1 && |
3069 |
assert(LogicOp.getValueType() == MVT::i1 && |
| 3070 |
"Can only handle logic operations on i1 values here."); |
3070 |
"Can only handle logic operations on i1 values here."); |
| 3071 |
SDLoc dl(LogicOp); |
3071 |
SDLoc dl(LogicOp); |
| 3072 |
SDValue LHS, RHS; |
3072 |
SDValue LHS, RHS; |
| 3073 |
|
3073 |
|
| 3074 |
// Special case: xor %a, -1 |
3074 |
// Special case: xor %a, -1 |
| 3075 |
bool IsBitwiseNegation = isBitwiseNot(LogicOp); |
3075 |
bool IsBitwiseNegation = isBitwiseNot(LogicOp); |
| 3076 |
|
3076 |
|
| 3077 |
// Produces a GPR sequence for each operand of the binary logic operation. |
3077 |
// Produces a GPR sequence for each operand of the binary logic operation. |
| 3078 |
// For SETCC, it produces the respective comparison, for TRUNCATE it truncates |
3078 |
// For SETCC, it produces the respective comparison, for TRUNCATE it truncates |
| 3079 |
// the value in a GPR and for logic operations, it will recursively produce |
3079 |
// the value in a GPR and for logic operations, it will recursively produce |
| 3080 |
// a GPR sequence for the operation. |
3080 |
// a GPR sequence for the operation. |
| 3081 |
auto getLogicOperand = [&] (SDValue Operand) -> SDValue { |
3081 |
auto getLogicOperand = [&] (SDValue Operand) -> SDValue { |
| 3082 |
unsigned OperandOpcode = Operand.getOpcode(); |
3082 |
unsigned OperandOpcode = Operand.getOpcode(); |
| 3083 |
if (OperandOpcode == ISD::SETCC) |
3083 |
if (OperandOpcode == ISD::SETCC) |
| 3084 |
return getSETCCInGPR(Operand, SetccInGPROpts::ZExtOrig); |
3084 |
return getSETCCInGPR(Operand, SetccInGPROpts::ZExtOrig); |
| 3085 |
else if (OperandOpcode == ISD::TRUNCATE) { |
3085 |
else if (OperandOpcode == ISD::TRUNCATE) { |
| 3086 |
SDValue InputOp = Operand.getOperand(0); |
3086 |
SDValue InputOp = Operand.getOperand(0); |
| 3087 |
EVT InVT = InputOp.getValueType(); |
3087 |
EVT InVT = InputOp.getValueType(); |
| 3088 |
return SDValue(CurDAG->getMachineNode(InVT == MVT::i32 ? PPC::RLDICL_32 : |
3088 |
return SDValue(CurDAG->getMachineNode(InVT == MVT::i32 ? PPC::RLDICL_32 : |
| 3089 |
PPC::RLDICL, dl, InVT, InputOp, |
3089 |
PPC::RLDICL, dl, InVT, InputOp, |
| 3090 |
S->getI64Imm(0, dl), |
3090 |
S->getI64Imm(0, dl), |
| 3091 |
S->getI64Imm(63, dl)), 0); |
3091 |
S->getI64Imm(63, dl)), 0); |
| 3092 |
} else if (ISD::isBitwiseLogicOp(OperandOpcode)) |
3092 |
} else if (ISD::isBitwiseLogicOp(OperandOpcode)) |
| 3093 |
return computeLogicOpInGPR(Operand); |
3093 |
return computeLogicOpInGPR(Operand); |
| 3094 |
return SDValue(); |
3094 |
return SDValue(); |
| 3095 |
}; |
3095 |
}; |
| 3096 |
LHS = getLogicOperand(LogicOp.getOperand(0)); |
3096 |
LHS = getLogicOperand(LogicOp.getOperand(0)); |
| 3097 |
RHS = getLogicOperand(LogicOp.getOperand(1)); |
3097 |
RHS = getLogicOperand(LogicOp.getOperand(1)); |
| 3098 |
|
3098 |
|
| 3099 |
// If a GPR sequence can't be produced for the LHS we can't proceed. |
3099 |
// If a GPR sequence can't be produced for the LHS we can't proceed. |
| 3100 |
// Not producing a GPR sequence for the RHS is only a problem if this isn't |
3100 |
// Not producing a GPR sequence for the RHS is only a problem if this isn't |
| 3101 |
// a bitwise negation operation. |
3101 |
// a bitwise negation operation. |
| 3102 |
if (!LHS || (!RHS && !IsBitwiseNegation)) |
3102 |
if (!LHS || (!RHS && !IsBitwiseNegation)) |
| 3103 |
return SDValue(); |
3103 |
return SDValue(); |
| 3104 |
|
3104 |
|
| 3105 |
NumLogicOpsOnComparison++; |
3105 |
NumLogicOpsOnComparison++; |
| 3106 |
|
3106 |
|
| 3107 |
// We will use the inputs as 64-bit values. |
3107 |
// We will use the inputs as 64-bit values. |
| 3108 |
if (LHS.getValueType() == MVT::i32) |
3108 |
if (LHS.getValueType() == MVT::i32) |
| 3109 |
LHS = addExtOrTrunc(LHS, ExtOrTruncConversion::Ext); |
3109 |
LHS = addExtOrTrunc(LHS, ExtOrTruncConversion::Ext); |
| 3110 |
if (!IsBitwiseNegation && RHS.getValueType() == MVT::i32) |
3110 |
if (!IsBitwiseNegation && RHS.getValueType() == MVT::i32) |
| 3111 |
RHS = addExtOrTrunc(RHS, ExtOrTruncConversion::Ext); |
3111 |
RHS = addExtOrTrunc(RHS, ExtOrTruncConversion::Ext); |
| 3112 |
|
3112 |
|
| 3113 |
unsigned NewOpc; |
3113 |
unsigned NewOpc; |
| 3114 |
switch (LogicOp.getOpcode()) { |
3114 |
switch (LogicOp.getOpcode()) { |
| 3115 |
default: llvm_unreachable("Unknown logic operation."); |
3115 |
default: llvm_unreachable("Unknown logic operation."); |
| 3116 |
case ISD::AND: NewOpc = PPC::AND8; break; |
3116 |
case ISD::AND: NewOpc = PPC::AND8; break; |
| 3117 |
case ISD::OR: NewOpc = PPC::OR8; break; |
3117 |
case ISD::OR: NewOpc = PPC::OR8; break; |
| 3118 |
case ISD::XOR: NewOpc = PPC::XOR8; break; |
3118 |
case ISD::XOR: NewOpc = PPC::XOR8; break; |
| 3119 |
} |
3119 |
} |
| 3120 |
|
3120 |
|
| 3121 |
if (IsBitwiseNegation) { |
3121 |
if (IsBitwiseNegation) { |
| 3122 |
RHS = S->getI64Imm(1, dl); |
3122 |
RHS = S->getI64Imm(1, dl); |
| 3123 |
NewOpc = PPC::XORI8; |
3123 |
NewOpc = PPC::XORI8; |
| 3124 |
} |
3124 |
} |
| 3125 |
|
3125 |
|
| 3126 |
return SDValue(CurDAG->getMachineNode(NewOpc, dl, MVT::i64, LHS, RHS), 0); |
3126 |
return SDValue(CurDAG->getMachineNode(NewOpc, dl, MVT::i64, LHS, RHS), 0); |
| 3127 |
|
3127 |
|
| 3128 |
} |
3128 |
} |
| 3129 |
|
3129 |
|
| 3130 |
/// If the value isn't guaranteed to be sign-extended to 64-bits, extend it. |
3130 |
/// If the value isn't guaranteed to be sign-extended to 64-bits, extend it. |
| 3131 |
/// Otherwise just reinterpret it as a 64-bit value. |
3131 |
/// Otherwise just reinterpret it as a 64-bit value. |
| 3132 |
/// Useful when emitting comparison code for 32-bit values without using |
3132 |
/// Useful when emitting comparison code for 32-bit values without using |
| 3133 |
/// the compare instruction (which only considers the lower 32-bits). |
3133 |
/// the compare instruction (which only considers the lower 32-bits). |
| 3134 |
SDValue IntegerCompareEliminator::signExtendInputIfNeeded(SDValue Input) { |
3134 |
SDValue IntegerCompareEliminator::signExtendInputIfNeeded(SDValue Input) { |
| 3135 |
assert(Input.getValueType() == MVT::i32 && |
3135 |
assert(Input.getValueType() == MVT::i32 && |
| 3136 |
"Can only sign-extend 32-bit values here."); |
3136 |
"Can only sign-extend 32-bit values here."); |
| 3137 |
unsigned Opc = Input.getOpcode(); |
3137 |
unsigned Opc = Input.getOpcode(); |
| 3138 |
|
3138 |
|
| 3139 |
// The value was sign extended and then truncated to 32-bits. No need to |
3139 |
// The value was sign extended and then truncated to 32-bits. No need to |
| 3140 |
// sign extend it again. |
3140 |
// sign extend it again. |
| 3141 |
if (Opc == ISD::TRUNCATE && |
3141 |
if (Opc == ISD::TRUNCATE && |
| 3142 |
(Input.getOperand(0).getOpcode() == ISD::AssertSext || |
3142 |
(Input.getOperand(0).getOpcode() == ISD::AssertSext || |
| 3143 |
Input.getOperand(0).getOpcode() == ISD::SIGN_EXTEND)) |
3143 |
Input.getOperand(0).getOpcode() == ISD::SIGN_EXTEND)) |
| 3144 |
return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); |
3144 |
return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); |
| 3145 |
|
3145 |
|
| 3146 |
LoadSDNode *InputLoad = dyn_cast(Input); |
3146 |
LoadSDNode *InputLoad = dyn_cast(Input); |
| 3147 |
// The input is a sign-extending load. All ppc sign-extending loads |
3147 |
// The input is a sign-extending load. All ppc sign-extending loads |
| 3148 |
// sign-extend to the full 64-bits. |
3148 |
// sign-extend to the full 64-bits. |
| 3149 |
if (InputLoad && InputLoad->getExtensionType() == ISD::SEXTLOAD) |
3149 |
if (InputLoad && InputLoad->getExtensionType() == ISD::SEXTLOAD) |
| 3150 |
return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); |
3150 |
return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); |
| 3151 |
|
3151 |
|
| 3152 |
ConstantSDNode *InputConst = dyn_cast(Input); |
3152 |
ConstantSDNode *InputConst = dyn_cast(Input); |
| 3153 |
// We don't sign-extend constants. |
3153 |
// We don't sign-extend constants. |
| 3154 |
if (InputConst) |
3154 |
if (InputConst) |
| 3155 |
return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); |
3155 |
return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); |
| 3156 |
|
3156 |
|
| 3157 |
SDLoc dl(Input); |
3157 |
SDLoc dl(Input); |
| 3158 |
SignExtensionsAdded++; |
3158 |
SignExtensionsAdded++; |
| 3159 |
return SDValue(CurDAG->getMachineNode(PPC::EXTSW_32_64, dl, |
3159 |
return SDValue(CurDAG->getMachineNode(PPC::EXTSW_32_64, dl, |
| 3160 |
MVT::i64, Input), 0); |
3160 |
MVT::i64, Input), 0); |
| 3161 |
} |
3161 |
} |
| 3162 |
|
3162 |
|
| 3163 |
/// If the value isn't guaranteed to be zero-extended to 64-bits, extend it. |
3163 |
/// If the value isn't guaranteed to be zero-extended to 64-bits, extend it. |
| 3164 |
/// Otherwise just reinterpret it as a 64-bit value. |
3164 |
/// Otherwise just reinterpret it as a 64-bit value. |
| 3165 |
/// Useful when emitting comparison code for 32-bit values without using |
3165 |
/// Useful when emitting comparison code for 32-bit values without using |
| 3166 |
/// the compare instruction (which only considers the lower 32-bits). |
3166 |
/// the compare instruction (which only considers the lower 32-bits). |
| 3167 |
SDValue IntegerCompareEliminator::zeroExtendInputIfNeeded(SDValue Input) { |
3167 |
SDValue IntegerCompareEliminator::zeroExtendInputIfNeeded(SDValue Input) { |
| 3168 |
assert(Input.getValueType() == MVT::i32 && |
3168 |
assert(Input.getValueType() == MVT::i32 && |
| 3169 |
"Can only zero-extend 32-bit values here."); |
3169 |
"Can only zero-extend 32-bit values here."); |
| 3170 |
unsigned Opc = Input.getOpcode(); |
3170 |
unsigned Opc = Input.getOpcode(); |
| 3171 |
|
3171 |
|
| 3172 |
// The only condition under which we can omit the actual extend instruction: |
3172 |
// The only condition under which we can omit the actual extend instruction: |
| 3173 |
// - The value is a positive constant |
3173 |
// - The value is a positive constant |
| 3174 |
// - The value comes from a load that isn't a sign-extending load |
3174 |
// - The value comes from a load that isn't a sign-extending load |
| 3175 |
// An ISD::TRUNCATE needs to be zero-extended unless it is fed by a zext. |
3175 |
// An ISD::TRUNCATE needs to be zero-extended unless it is fed by a zext. |
| 3176 |
bool IsTruncateOfZExt = Opc == ISD::TRUNCATE && |
3176 |
bool IsTruncateOfZExt = Opc == ISD::TRUNCATE && |
| 3177 |
(Input.getOperand(0).getOpcode() == ISD::AssertZext || |
3177 |
(Input.getOperand(0).getOpcode() == ISD::AssertZext || |
| 3178 |
Input.getOperand(0).getOpcode() == ISD::ZERO_EXTEND); |
3178 |
Input.getOperand(0).getOpcode() == ISD::ZERO_EXTEND); |
| 3179 |
if (IsTruncateOfZExt) |
3179 |
if (IsTruncateOfZExt) |
| 3180 |
return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); |
3180 |
return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); |
| 3181 |
|
3181 |
|
| 3182 |
ConstantSDNode *InputConst = dyn_cast(Input); |
3182 |
ConstantSDNode *InputConst = dyn_cast(Input); |
| 3183 |
if (InputConst && InputConst->getSExtValue() >= 0) |
3183 |
if (InputConst && InputConst->getSExtValue() >= 0) |
| 3184 |
return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); |
3184 |
return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); |
| 3185 |
|
3185 |
|
| 3186 |
LoadSDNode *InputLoad = dyn_cast(Input); |
3186 |
LoadSDNode *InputLoad = dyn_cast(Input); |
| 3187 |
// The input is a load that doesn't sign-extend (it will be zero-extended). |
3187 |
// The input is a load that doesn't sign-extend (it will be zero-extended). |
| 3188 |
if (InputLoad && InputLoad->getExtensionType() != ISD::SEXTLOAD) |
3188 |
if (InputLoad && InputLoad->getExtensionType() != ISD::SEXTLOAD) |
| 3189 |
return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); |
3189 |
return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); |
| 3190 |
|
3190 |
|
| 3191 |
// None of the above, need to zero-extend. |
3191 |
// None of the above, need to zero-extend. |
| 3192 |
SDLoc dl(Input); |
3192 |
SDLoc dl(Input); |
| 3193 |
ZeroExtensionsAdded++; |
3193 |
ZeroExtensionsAdded++; |
| 3194 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL_32_64, dl, MVT::i64, Input, |
3194 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL_32_64, dl, MVT::i64, Input, |
| 3195 |
S->getI64Imm(0, dl), |
3195 |
S->getI64Imm(0, dl), |
| 3196 |
S->getI64Imm(32, dl)), 0); |
3196 |
S->getI64Imm(32, dl)), 0); |
| 3197 |
} |
3197 |
} |
| 3198 |
|
3198 |
|
| 3199 |
// Handle a 32-bit value in a 64-bit register and vice-versa. These are of |
3199 |
// Handle a 32-bit value in a 64-bit register and vice-versa. These are of |
| 3200 |
// course not actual zero/sign extensions that will generate machine code, |
3200 |
// course not actual zero/sign extensions that will generate machine code, |
| 3201 |
// they're just a way to reinterpret a 32 bit value in a register as a |
3201 |
// they're just a way to reinterpret a 32 bit value in a register as a |
| 3202 |
// 64 bit value and vice-versa. |
3202 |
// 64 bit value and vice-versa. |
| 3203 |
SDValue IntegerCompareEliminator::addExtOrTrunc(SDValue NatWidthRes, |
3203 |
SDValue IntegerCompareEliminator::addExtOrTrunc(SDValue NatWidthRes, |
| 3204 |
ExtOrTruncConversion Conv) { |
3204 |
ExtOrTruncConversion Conv) { |
| 3205 |
SDLoc dl(NatWidthRes); |
3205 |
SDLoc dl(NatWidthRes); |
| 3206 |
|
3206 |
|
| 3207 |
// For reinterpreting 32-bit values as 64 bit values, we generate |
3207 |
// For reinterpreting 32-bit values as 64 bit values, we generate |
| 3208 |
// INSERT_SUBREG IMPLICIT_DEF:i64, , TargetConstant:i32<1> |
3208 |
// INSERT_SUBREG IMPLICIT_DEF:i64, , TargetConstant:i32<1> |
| 3209 |
if (Conv == ExtOrTruncConversion::Ext) { |
3209 |
if (Conv == ExtOrTruncConversion::Ext) { |
| 3210 |
SDValue ImDef(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, MVT::i64), 0); |
3210 |
SDValue ImDef(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, MVT::i64), 0); |
| 3211 |
SDValue SubRegIdx = |
3211 |
SDValue SubRegIdx = |
| 3212 |
CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); |
3212 |
CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); |
| 3213 |
return SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, MVT::i64, |
3213 |
return SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, MVT::i64, |
| 3214 |
ImDef, NatWidthRes, SubRegIdx), 0); |
3214 |
ImDef, NatWidthRes, SubRegIdx), 0); |
| 3215 |
} |
3215 |
} |
| 3216 |
|
3216 |
|
| 3217 |
assert(Conv == ExtOrTruncConversion::Trunc && |
3217 |
assert(Conv == ExtOrTruncConversion::Trunc && |
| 3218 |
"Unknown convertion between 32 and 64 bit values."); |
3218 |
"Unknown convertion between 32 and 64 bit values."); |
| 3219 |
// For reinterpreting 64-bit values as 32-bit values, we just need to |
3219 |
// For reinterpreting 64-bit values as 32-bit values, we just need to |
| 3220 |
// EXTRACT_SUBREG (i.e. extract the low word). |
3220 |
// EXTRACT_SUBREG (i.e. extract the low word). |
| 3221 |
SDValue SubRegIdx = |
3221 |
SDValue SubRegIdx = |
| 3222 |
CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); |
3222 |
CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); |
| 3223 |
return SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, MVT::i32, |
3223 |
return SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, MVT::i32, |
| 3224 |
NatWidthRes, SubRegIdx), 0); |
3224 |
NatWidthRes, SubRegIdx), 0); |
| 3225 |
} |
3225 |
} |
| 3226 |
|
3226 |
|
| 3227 |
// Produce a GPR sequence for compound comparisons (<=, >=) against zero. |
3227 |
// Produce a GPR sequence for compound comparisons (<=, >=) against zero. |
| 3228 |
// Handle both zero-extensions and sign-extensions. |
3228 |
// Handle both zero-extensions and sign-extensions. |
| 3229 |
SDValue |
3229 |
SDValue |
| 3230 |
IntegerCompareEliminator::getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl, |
3230 |
IntegerCompareEliminator::getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl, |
| 3231 |
ZeroCompare CmpTy) { |
3231 |
ZeroCompare CmpTy) { |
| 3232 |
EVT InVT = LHS.getValueType(); |
3232 |
EVT InVT = LHS.getValueType(); |
| 3233 |
bool Is32Bit = InVT == MVT::i32; |
3233 |
bool Is32Bit = InVT == MVT::i32; |
| 3234 |
SDValue ToExtend; |
3234 |
SDValue ToExtend; |
| 3235 |
|
3235 |
|
| 3236 |
// Produce the value that needs to be either zero or sign extended. |
3236 |
// Produce the value that needs to be either zero or sign extended. |
| 3237 |
switch (CmpTy) { |
3237 |
switch (CmpTy) { |
| 3238 |
case ZeroCompare::GEZExt: |
3238 |
case ZeroCompare::GEZExt: |
| 3239 |
case ZeroCompare::GESExt: |
3239 |
case ZeroCompare::GESExt: |
| 3240 |
ToExtend = SDValue(CurDAG->getMachineNode(Is32Bit ? PPC::NOR : PPC::NOR8, |
3240 |
ToExtend = SDValue(CurDAG->getMachineNode(Is32Bit ? PPC::NOR : PPC::NOR8, |
| 3241 |
dl, InVT, LHS, LHS), 0); |
3241 |
dl, InVT, LHS, LHS), 0); |
| 3242 |
break; |
3242 |
break; |
| 3243 |
case ZeroCompare::LEZExt: |
3243 |
case ZeroCompare::LEZExt: |
| 3244 |
case ZeroCompare::LESExt: { |
3244 |
case ZeroCompare::LESExt: { |
| 3245 |
if (Is32Bit) { |
3245 |
if (Is32Bit) { |
| 3246 |
// Upper 32 bits cannot be undefined for this sequence. |
3246 |
// Upper 32 bits cannot be undefined for this sequence. |
| 3247 |
LHS = signExtendInputIfNeeded(LHS); |
3247 |
LHS = signExtendInputIfNeeded(LHS); |
| 3248 |
SDValue Neg = |
3248 |
SDValue Neg = |
| 3249 |
SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); |
3249 |
SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); |
| 3250 |
ToExtend = |
3250 |
ToExtend = |
| 3251 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3251 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
| 3252 |
Neg, S->getI64Imm(1, dl), |
3252 |
Neg, S->getI64Imm(1, dl), |
| 3253 |
S->getI64Imm(63, dl)), 0); |
3253 |
S->getI64Imm(63, dl)), 0); |
| 3254 |
} else { |
3254 |
} else { |
| 3255 |
SDValue Addi = |
3255 |
SDValue Addi = |
| 3256 |
SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, |
3256 |
SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, |
| 3257 |
S->getI64Imm(~0ULL, dl)), 0); |
3257 |
S->getI64Imm(~0ULL, dl)), 0); |
| 3258 |
ToExtend = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, |
3258 |
ToExtend = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, |
| 3259 |
Addi, LHS), 0); |
3259 |
Addi, LHS), 0); |
| 3260 |
} |
3260 |
} |
| 3261 |
break; |
3261 |
break; |
| 3262 |
} |
3262 |
} |
| 3263 |
} |
3263 |
} |
| 3264 |
|
3264 |
|
| 3265 |
// For 64-bit sequences, the extensions are the same for the GE/LE cases. |
3265 |
// For 64-bit sequences, the extensions are the same for the GE/LE cases. |
| 3266 |
if (!Is32Bit && |
3266 |
if (!Is32Bit && |
| 3267 |
(CmpTy == ZeroCompare::GEZExt || CmpTy == ZeroCompare::LEZExt)) |
3267 |
(CmpTy == ZeroCompare::GEZExt || CmpTy == ZeroCompare::LEZExt)) |
| 3268 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3268 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
| 3269 |
ToExtend, S->getI64Imm(1, dl), |
3269 |
ToExtend, S->getI64Imm(1, dl), |
| 3270 |
S->getI64Imm(63, dl)), 0); |
3270 |
S->getI64Imm(63, dl)), 0); |
| 3271 |
if (!Is32Bit && |
3271 |
if (!Is32Bit && |
| 3272 |
(CmpTy == ZeroCompare::GESExt || CmpTy == ZeroCompare::LESExt)) |
3272 |
(CmpTy == ZeroCompare::GESExt || CmpTy == ZeroCompare::LESExt)) |
| 3273 |
return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, ToExtend, |
3273 |
return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, ToExtend, |
| 3274 |
S->getI64Imm(63, dl)), 0); |
3274 |
S->getI64Imm(63, dl)), 0); |
| 3275 |
|
3275 |
|
| 3276 |
assert(Is32Bit && "Should have handled the 32-bit sequences above."); |
3276 |
assert(Is32Bit && "Should have handled the 32-bit sequences above."); |
| 3277 |
// For 32-bit sequences, the extensions differ between GE/LE cases. |
3277 |
// For 32-bit sequences, the extensions differ between GE/LE cases. |
| 3278 |
switch (CmpTy) { |
3278 |
switch (CmpTy) { |
| 3279 |
case ZeroCompare::GEZExt: { |
3279 |
case ZeroCompare::GEZExt: { |
| 3280 |
SDValue ShiftOps[] = { ToExtend, S->getI32Imm(1, dl), S->getI32Imm(31, dl), |
3280 |
SDValue ShiftOps[] = { ToExtend, S->getI32Imm(1, dl), S->getI32Imm(31, dl), |
| 3281 |
S->getI32Imm(31, dl) }; |
3281 |
S->getI32Imm(31, dl) }; |
| 3282 |
return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, |
3282 |
return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, |
| 3283 |
ShiftOps), 0); |
3283 |
ShiftOps), 0); |
| 3284 |
} |
3284 |
} |
| 3285 |
case ZeroCompare::GESExt: |
3285 |
case ZeroCompare::GESExt: |
| 3286 |
return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, ToExtend, |
3286 |
return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, ToExtend, |
| 3287 |
S->getI32Imm(31, dl)), 0); |
3287 |
S->getI32Imm(31, dl)), 0); |
| 3288 |
case ZeroCompare::LEZExt: |
3288 |
case ZeroCompare::LEZExt: |
| 3289 |
return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, ToExtend, |
3289 |
return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, ToExtend, |
| 3290 |
S->getI32Imm(1, dl)), 0); |
3290 |
S->getI32Imm(1, dl)), 0); |
| 3291 |
case ZeroCompare::LESExt: |
3291 |
case ZeroCompare::LESExt: |
| 3292 |
return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, ToExtend, |
3292 |
return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, ToExtend, |
| 3293 |
S->getI32Imm(-1, dl)), 0); |
3293 |
S->getI32Imm(-1, dl)), 0); |
| 3294 |
} |
3294 |
} |
| 3295 |
|
3295 |
|
| 3296 |
// The above case covers all the enumerators so it can't have a default clause |
3296 |
// The above case covers all the enumerators so it can't have a default clause |
| 3297 |
// to avoid compiler warnings. |
3297 |
// to avoid compiler warnings. |
| 3298 |
llvm_unreachable("Unknown zero-comparison type."); |
3298 |
llvm_unreachable("Unknown zero-comparison type."); |
| 3299 |
} |
3299 |
} |
| 3300 |
|
3300 |
|
| 3301 |
/// Produces a zero-extended result of comparing two 32-bit values according to |
3301 |
/// Produces a zero-extended result of comparing two 32-bit values according to |
| 3302 |
/// the passed condition code. |
3302 |
/// the passed condition code. |
| 3303 |
SDValue |
3303 |
SDValue |
| 3304 |
IntegerCompareEliminator::get32BitZExtCompare(SDValue LHS, SDValue RHS, |
3304 |
IntegerCompareEliminator::get32BitZExtCompare(SDValue LHS, SDValue RHS, |
| 3305 |
ISD::CondCode CC, |
3305 |
ISD::CondCode CC, |
| 3306 |
int64_t RHSValue, SDLoc dl) { |
3306 |
int64_t RHSValue, SDLoc dl) { |
| 3307 |
if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 || |
3307 |
if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 || |
| 3308 |
CmpInGPR == ICGPR_ZextI64 || CmpInGPR == ICGPR_Sext) |
3308 |
CmpInGPR == ICGPR_ZextI64 || CmpInGPR == ICGPR_Sext) |
| 3309 |
return SDValue(); |
3309 |
return SDValue(); |
| 3310 |
bool IsRHSZero = RHSValue == 0; |
3310 |
bool IsRHSZero = RHSValue == 0; |
| 3311 |
bool IsRHSOne = RHSValue == 1; |
3311 |
bool IsRHSOne = RHSValue == 1; |
| 3312 |
bool IsRHSNegOne = RHSValue == -1LL; |
3312 |
bool IsRHSNegOne = RHSValue == -1LL; |
| 3313 |
switch (CC) { |
3313 |
switch (CC) { |
| 3314 |
default: return SDValue(); |
3314 |
default: return SDValue(); |
| 3315 |
case ISD::SETEQ: { |
3315 |
case ISD::SETEQ: { |
| 3316 |
// (zext (setcc %a, %b, seteq)) -> (lshr (cntlzw (xor %a, %b)), 5) |
3316 |
// (zext (setcc %a, %b, seteq)) -> (lshr (cntlzw (xor %a, %b)), 5) |
| 3317 |
// (zext (setcc %a, 0, seteq)) -> (lshr (cntlzw %a), 5) |
3317 |
// (zext (setcc %a, 0, seteq)) -> (lshr (cntlzw %a), 5) |
| 3318 |
SDValue Xor = IsRHSZero ? LHS : |
3318 |
SDValue Xor = IsRHSZero ? LHS : |
| 3319 |
SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); |
3319 |
SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); |
| 3320 |
SDValue Clz = |
3320 |
SDValue Clz = |
| 3321 |
SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); |
3321 |
SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); |
| 3322 |
SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), |
3322 |
SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), |
| 3323 |
S->getI32Imm(31, dl) }; |
3323 |
S->getI32Imm(31, dl) }; |
| 3324 |
return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, |
3324 |
return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, |
| 3325 |
ShiftOps), 0); |
3325 |
ShiftOps), 0); |
| 3326 |
} |
3326 |
} |
| 3327 |
case ISD::SETNE: { |
3327 |
case ISD::SETNE: { |
| 3328 |
// (zext (setcc %a, %b, setne)) -> (xor (lshr (cntlzw (xor %a, %b)), 5), 1) |
3328 |
// (zext (setcc %a, %b, setne)) -> (xor (lshr (cntlzw (xor %a, %b)), 5), 1) |
| 3329 |
// (zext (setcc %a, 0, setne)) -> (xor (lshr (cntlzw %a), 5), 1) |
3329 |
// (zext (setcc %a, 0, setne)) -> (xor (lshr (cntlzw %a), 5), 1) |
| 3330 |
SDValue Xor = IsRHSZero ? LHS : |
3330 |
SDValue Xor = IsRHSZero ? LHS : |
| 3331 |
SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); |
3331 |
SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); |
| 3332 |
SDValue Clz = |
3332 |
SDValue Clz = |
| 3333 |
SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); |
3333 |
SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); |
| 3334 |
SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), |
3334 |
SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), |
| 3335 |
S->getI32Imm(31, dl) }; |
3335 |
S->getI32Imm(31, dl) }; |
| 3336 |
SDValue Shift = |
3336 |
SDValue Shift = |
| 3337 |
SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0); |
3337 |
SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0); |
| 3338 |
return SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift, |
3338 |
return SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift, |
| 3339 |
S->getI32Imm(1, dl)), 0); |
3339 |
S->getI32Imm(1, dl)), 0); |
| 3340 |
} |
3340 |
} |
| 3341 |
case ISD::SETGE: { |
3341 |
case ISD::SETGE: { |
| 3342 |
// (zext (setcc %a, %b, setge)) -> (xor (lshr (sub %a, %b), 63), 1) |
3342 |
// (zext (setcc %a, %b, setge)) -> (xor (lshr (sub %a, %b), 63), 1) |
| 3343 |
// (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 31) |
3343 |
// (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 31) |
| 3344 |
if(IsRHSZero) |
3344 |
if(IsRHSZero) |
| 3345 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); |
3345 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); |
| 3346 |
|
3346 |
|
| 3347 |
// Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a) |
3347 |
// Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a) |
| 3348 |
// by swapping inputs and falling through. |
3348 |
// by swapping inputs and falling through. |
| 3349 |
std::swap(LHS, RHS); |
3349 |
std::swap(LHS, RHS); |
| 3350 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
3350 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
| 3351 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
3351 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
| 3352 |
[[fallthrough]]; |
3352 |
[[fallthrough]]; |
| 3353 |
} |
3353 |
} |
| 3354 |
case ISD::SETLE: { |
3354 |
case ISD::SETLE: { |
| 3355 |
if (CmpInGPR == ICGPR_NonExtIn) |
3355 |
if (CmpInGPR == ICGPR_NonExtIn) |
| 3356 |
return SDValue(); |
3356 |
return SDValue(); |
| 3357 |
// (zext (setcc %a, %b, setle)) -> (xor (lshr (sub %b, %a), 63), 1) |
3357 |
// (zext (setcc %a, %b, setle)) -> (xor (lshr (sub %b, %a), 63), 1) |
| 3358 |
// (zext (setcc %a, 0, setle)) -> (xor (lshr (- %a), 63), 1) |
3358 |
// (zext (setcc %a, 0, setle)) -> (xor (lshr (- %a), 63), 1) |
| 3359 |
if(IsRHSZero) { |
3359 |
if(IsRHSZero) { |
| 3360 |
if (CmpInGPR == ICGPR_NonExtIn) |
3360 |
if (CmpInGPR == ICGPR_NonExtIn) |
| 3361 |
return SDValue(); |
3361 |
return SDValue(); |
| 3362 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); |
3362 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); |
| 3363 |
} |
3363 |
} |
| 3364 |
|
3364 |
|
| 3365 |
// The upper 32-bits of the register can't be undefined for this sequence. |
3365 |
// The upper 32-bits of the register can't be undefined for this sequence. |
| 3366 |
LHS = signExtendInputIfNeeded(LHS); |
3366 |
LHS = signExtendInputIfNeeded(LHS); |
| 3367 |
RHS = signExtendInputIfNeeded(RHS); |
3367 |
RHS = signExtendInputIfNeeded(RHS); |
| 3368 |
SDValue Sub = |
3368 |
SDValue Sub = |
| 3369 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); |
3369 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); |
| 3370 |
SDValue Shift = |
3370 |
SDValue Shift = |
| 3371 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Sub, |
3371 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Sub, |
| 3372 |
S->getI64Imm(1, dl), S->getI64Imm(63, dl)), |
3372 |
S->getI64Imm(1, dl), S->getI64Imm(63, dl)), |
| 3373 |
0); |
3373 |
0); |
| 3374 |
return |
3374 |
return |
| 3375 |
SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, |
3375 |
SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, |
| 3376 |
MVT::i64, Shift, S->getI32Imm(1, dl)), 0); |
3376 |
MVT::i64, Shift, S->getI32Imm(1, dl)), 0); |
| 3377 |
} |
3377 |
} |
| 3378 |
case ISD::SETGT: { |
3378 |
case ISD::SETGT: { |
| 3379 |
// (zext (setcc %a, %b, setgt)) -> (lshr (sub %b, %a), 63) |
3379 |
// (zext (setcc %a, %b, setgt)) -> (lshr (sub %b, %a), 63) |
| 3380 |
// (zext (setcc %a, -1, setgt)) -> (lshr (~ %a), 31) |
3380 |
// (zext (setcc %a, -1, setgt)) -> (lshr (~ %a), 31) |
| 3381 |
// (zext (setcc %a, 0, setgt)) -> (lshr (- %a), 63) |
3381 |
// (zext (setcc %a, 0, setgt)) -> (lshr (- %a), 63) |
| 3382 |
// Handle SETLT -1 (which is equivalent to SETGE 0). |
3382 |
// Handle SETLT -1 (which is equivalent to SETGE 0). |
| 3383 |
if (IsRHSNegOne) |
3383 |
if (IsRHSNegOne) |
| 3384 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); |
3384 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); |
| 3385 |
|
3385 |
|
| 3386 |
if (IsRHSZero) { |
3386 |
if (IsRHSZero) { |
| 3387 |
if (CmpInGPR == ICGPR_NonExtIn) |
3387 |
if (CmpInGPR == ICGPR_NonExtIn) |
| 3388 |
return SDValue(); |
3388 |
return SDValue(); |
| 3389 |
// The upper 32-bits of the register can't be undefined for this sequence. |
3389 |
// The upper 32-bits of the register can't be undefined for this sequence. |
| 3390 |
LHS = signExtendInputIfNeeded(LHS); |
3390 |
LHS = signExtendInputIfNeeded(LHS); |
| 3391 |
RHS = signExtendInputIfNeeded(RHS); |
3391 |
RHS = signExtendInputIfNeeded(RHS); |
| 3392 |
SDValue Neg = |
3392 |
SDValue Neg = |
| 3393 |
SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); |
3393 |
SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); |
| 3394 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3394 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
| 3395 |
Neg, S->getI32Imm(1, dl), S->getI32Imm(63, dl)), 0); |
3395 |
Neg, S->getI32Imm(1, dl), S->getI32Imm(63, dl)), 0); |
| 3396 |
} |
3396 |
} |
| 3397 |
// Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as |
3397 |
// Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as |
| 3398 |
// (%b < %a) by swapping inputs and falling through. |
3398 |
// (%b < %a) by swapping inputs and falling through. |
| 3399 |
std::swap(LHS, RHS); |
3399 |
std::swap(LHS, RHS); |
| 3400 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
3400 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
| 3401 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
3401 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
| 3402 |
IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; |
3402 |
IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; |
| 3403 |
[[fallthrough]]; |
3403 |
[[fallthrough]]; |
| 3404 |
} |
3404 |
} |
| 3405 |
case ISD::SETLT: { |
3405 |
case ISD::SETLT: { |
| 3406 |
// (zext (setcc %a, %b, setlt)) -> (lshr (sub %a, %b), 63) |
3406 |
// (zext (setcc %a, %b, setlt)) -> (lshr (sub %a, %b), 63) |
| 3407 |
// (zext (setcc %a, 1, setlt)) -> (xor (lshr (- %a), 63), 1) |
3407 |
// (zext (setcc %a, 1, setlt)) -> (xor (lshr (- %a), 63), 1) |
| 3408 |
// (zext (setcc %a, 0, setlt)) -> (lshr %a, 31) |
3408 |
// (zext (setcc %a, 0, setlt)) -> (lshr %a, 31) |
| 3409 |
// Handle SETLT 1 (which is equivalent to SETLE 0). |
3409 |
// Handle SETLT 1 (which is equivalent to SETLE 0). |
| 3410 |
if (IsRHSOne) { |
3410 |
if (IsRHSOne) { |
| 3411 |
if (CmpInGPR == ICGPR_NonExtIn) |
3411 |
if (CmpInGPR == ICGPR_NonExtIn) |
| 3412 |
return SDValue(); |
3412 |
return SDValue(); |
| 3413 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); |
3413 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); |
| 3414 |
} |
3414 |
} |
| 3415 |
|
3415 |
|
| 3416 |
if (IsRHSZero) { |
3416 |
if (IsRHSZero) { |
| 3417 |
SDValue ShiftOps[] = { LHS, S->getI32Imm(1, dl), S->getI32Imm(31, dl), |
3417 |
SDValue ShiftOps[] = { LHS, S->getI32Imm(1, dl), S->getI32Imm(31, dl), |
| 3418 |
S->getI32Imm(31, dl) }; |
3418 |
S->getI32Imm(31, dl) }; |
| 3419 |
return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, |
3419 |
return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, |
| 3420 |
ShiftOps), 0); |
3420 |
ShiftOps), 0); |
| 3421 |
} |
3421 |
} |
| 3422 |
|
3422 |
|
| 3423 |
if (CmpInGPR == ICGPR_NonExtIn) |
3423 |
if (CmpInGPR == ICGPR_NonExtIn) |
| 3424 |
return SDValue(); |
3424 |
return SDValue(); |
| 3425 |
// The upper 32-bits of the register can't be undefined for this sequence. |
3425 |
// The upper 32-bits of the register can't be undefined for this sequence. |
| 3426 |
LHS = signExtendInputIfNeeded(LHS); |
3426 |
LHS = signExtendInputIfNeeded(LHS); |
| 3427 |
RHS = signExtendInputIfNeeded(RHS); |
3427 |
RHS = signExtendInputIfNeeded(RHS); |
| 3428 |
SDValue SUBFNode = |
3428 |
SDValue SUBFNode = |
| 3429 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); |
3429 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); |
| 3430 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3430 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
| 3431 |
SUBFNode, S->getI64Imm(1, dl), |
3431 |
SUBFNode, S->getI64Imm(1, dl), |
| 3432 |
S->getI64Imm(63, dl)), 0); |
3432 |
S->getI64Imm(63, dl)), 0); |
| 3433 |
} |
3433 |
} |
| 3434 |
case ISD::SETUGE: |
3434 |
case ISD::SETUGE: |
| 3435 |
// (zext (setcc %a, %b, setuge)) -> (xor (lshr (sub %b, %a), 63), 1) |
3435 |
// (zext (setcc %a, %b, setuge)) -> (xor (lshr (sub %b, %a), 63), 1) |
| 3436 |
// (zext (setcc %a, %b, setule)) -> (xor (lshr (sub %a, %b), 63), 1) |
3436 |
// (zext (setcc %a, %b, setule)) -> (xor (lshr (sub %a, %b), 63), 1) |
| 3437 |
std::swap(LHS, RHS); |
3437 |
std::swap(LHS, RHS); |
| 3438 |
[[fallthrough]]; |
3438 |
[[fallthrough]]; |
| 3439 |
case ISD::SETULE: { |
3439 |
case ISD::SETULE: { |
| 3440 |
if (CmpInGPR == ICGPR_NonExtIn) |
3440 |
if (CmpInGPR == ICGPR_NonExtIn) |
| 3441 |
return SDValue(); |
3441 |
return SDValue(); |
| 3442 |
// The upper 32-bits of the register can't be undefined for this sequence. |
3442 |
// The upper 32-bits of the register can't be undefined for this sequence. |
| 3443 |
LHS = zeroExtendInputIfNeeded(LHS); |
3443 |
LHS = zeroExtendInputIfNeeded(LHS); |
| 3444 |
RHS = zeroExtendInputIfNeeded(RHS); |
3444 |
RHS = zeroExtendInputIfNeeded(RHS); |
| 3445 |
SDValue Subtract = |
3445 |
SDValue Subtract = |
| 3446 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); |
3446 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); |
| 3447 |
SDValue SrdiNode = |
3447 |
SDValue SrdiNode = |
| 3448 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3448 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
| 3449 |
Subtract, S->getI64Imm(1, dl), |
3449 |
Subtract, S->getI64Imm(1, dl), |
| 3450 |
S->getI64Imm(63, dl)), 0); |
3450 |
S->getI64Imm(63, dl)), 0); |
| 3451 |
return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, SrdiNode, |
3451 |
return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, SrdiNode, |
| 3452 |
S->getI32Imm(1, dl)), 0); |
3452 |
S->getI32Imm(1, dl)), 0); |
| 3453 |
} |
3453 |
} |
| 3454 |
case ISD::SETUGT: |
3454 |
case ISD::SETUGT: |
| 3455 |
// (zext (setcc %a, %b, setugt)) -> (lshr (sub %b, %a), 63) |
3455 |
// (zext (setcc %a, %b, setugt)) -> (lshr (sub %b, %a), 63) |
| 3456 |
// (zext (setcc %a, %b, setult)) -> (lshr (sub %a, %b), 63) |
3456 |
// (zext (setcc %a, %b, setult)) -> (lshr (sub %a, %b), 63) |
| 3457 |
std::swap(LHS, RHS); |
3457 |
std::swap(LHS, RHS); |
| 3458 |
[[fallthrough]]; |
3458 |
[[fallthrough]]; |
| 3459 |
case ISD::SETULT: { |
3459 |
case ISD::SETULT: { |
| 3460 |
if (CmpInGPR == ICGPR_NonExtIn) |
3460 |
if (CmpInGPR == ICGPR_NonExtIn) |
| 3461 |
return SDValue(); |
3461 |
return SDValue(); |
| 3462 |
// The upper 32-bits of the register can't be undefined for this sequence. |
3462 |
// The upper 32-bits of the register can't be undefined for this sequence. |
| 3463 |
LHS = zeroExtendInputIfNeeded(LHS); |
3463 |
LHS = zeroExtendInputIfNeeded(LHS); |
| 3464 |
RHS = zeroExtendInputIfNeeded(RHS); |
3464 |
RHS = zeroExtendInputIfNeeded(RHS); |
| 3465 |
SDValue Subtract = |
3465 |
SDValue Subtract = |
| 3466 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); |
3466 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); |
| 3467 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3467 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
| 3468 |
Subtract, S->getI64Imm(1, dl), |
3468 |
Subtract, S->getI64Imm(1, dl), |
| 3469 |
S->getI64Imm(63, dl)), 0); |
3469 |
S->getI64Imm(63, dl)), 0); |
| 3470 |
} |
3470 |
} |
| 3471 |
} |
3471 |
} |
| 3472 |
} |
3472 |
} |
| 3473 |
|
3473 |
|
| 3474 |
/// Produces a sign-extended result of comparing two 32-bit values according to |
3474 |
/// Produces a sign-extended result of comparing two 32-bit values according to |
| 3475 |
/// the passed condition code. |
3475 |
/// the passed condition code. |
| 3476 |
SDValue |
3476 |
SDValue |
| 3477 |
IntegerCompareEliminator::get32BitSExtCompare(SDValue LHS, SDValue RHS, |
3477 |
IntegerCompareEliminator::get32BitSExtCompare(SDValue LHS, SDValue RHS, |
| 3478 |
ISD::CondCode CC, |
3478 |
ISD::CondCode CC, |
| 3479 |
int64_t RHSValue, SDLoc dl) { |
3479 |
int64_t RHSValue, SDLoc dl) { |
| 3480 |
if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 || |
3480 |
if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 || |
| 3481 |
CmpInGPR == ICGPR_ZextI64 || CmpInGPR == ICGPR_Zext) |
3481 |
CmpInGPR == ICGPR_ZextI64 || CmpInGPR == ICGPR_Zext) |
| 3482 |
return SDValue(); |
3482 |
return SDValue(); |
| 3483 |
bool IsRHSZero = RHSValue == 0; |
3483 |
bool IsRHSZero = RHSValue == 0; |
| 3484 |
bool IsRHSOne = RHSValue == 1; |
3484 |
bool IsRHSOne = RHSValue == 1; |
| 3485 |
bool IsRHSNegOne = RHSValue == -1LL; |
3485 |
bool IsRHSNegOne = RHSValue == -1LL; |
| 3486 |
|
3486 |
|
| 3487 |
switch (CC) { |
3487 |
switch (CC) { |
| 3488 |
default: return SDValue(); |
3488 |
default: return SDValue(); |
| 3489 |
case ISD::SETEQ: { |
3489 |
case ISD::SETEQ: { |
| 3490 |
// (sext (setcc %a, %b, seteq)) -> |
3490 |
// (sext (setcc %a, %b, seteq)) -> |
| 3491 |
// (ashr (shl (ctlz (xor %a, %b)), 58), 63) |
3491 |
// (ashr (shl (ctlz (xor %a, %b)), 58), 63) |
| 3492 |
// (sext (setcc %a, 0, seteq)) -> |
3492 |
// (sext (setcc %a, 0, seteq)) -> |
| 3493 |
// (ashr (shl (ctlz %a), 58), 63) |
3493 |
// (ashr (shl (ctlz %a), 58), 63) |
| 3494 |
SDValue CountInput = IsRHSZero ? LHS : |
3494 |
SDValue CountInput = IsRHSZero ? LHS : |
| 3495 |
SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); |
3495 |
SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); |
| 3496 |
SDValue Cntlzw = |
3496 |
SDValue Cntlzw = |
| 3497 |
SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, CountInput), 0); |
3497 |
SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, CountInput), 0); |
| 3498 |
SDValue SHLOps[] = { Cntlzw, S->getI32Imm(27, dl), |
3498 |
SDValue SHLOps[] = { Cntlzw, S->getI32Imm(27, dl), |
| 3499 |
S->getI32Imm(5, dl), S->getI32Imm(31, dl) }; |
3499 |
S->getI32Imm(5, dl), S->getI32Imm(31, dl) }; |
| 3500 |
SDValue Slwi = |
3500 |
SDValue Slwi = |
| 3501 |
SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, SHLOps), 0); |
3501 |
SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, SHLOps), 0); |
| 3502 |
return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Slwi), 0); |
3502 |
return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Slwi), 0); |
| 3503 |
} |
3503 |
} |
| 3504 |
case ISD::SETNE: { |
3504 |
case ISD::SETNE: { |
| 3505 |
// Bitwise xor the operands, count leading zeros, shift right by 5 bits and |
3505 |
// Bitwise xor the operands, count leading zeros, shift right by 5 bits and |
| 3506 |
// flip the bit, finally take 2's complement. |
3506 |
// flip the bit, finally take 2's complement. |
| 3507 |
// (sext (setcc %a, %b, setne)) -> |
3507 |
// (sext (setcc %a, %b, setne)) -> |
| 3508 |
// (neg (xor (lshr (ctlz (xor %a, %b)), 5), 1)) |
3508 |
// (neg (xor (lshr (ctlz (xor %a, %b)), 5), 1)) |
| 3509 |
// Same as above, but the first xor is not needed. |
3509 |
// Same as above, but the first xor is not needed. |
| 3510 |
// (sext (setcc %a, 0, setne)) -> |
3510 |
// (sext (setcc %a, 0, setne)) -> |
| 3511 |
// (neg (xor (lshr (ctlz %a), 5), 1)) |
3511 |
// (neg (xor (lshr (ctlz %a), 5), 1)) |
| 3512 |
SDValue Xor = IsRHSZero ? LHS : |
3512 |
SDValue Xor = IsRHSZero ? LHS : |
| 3513 |
SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); |
3513 |
SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); |
| 3514 |
SDValue Clz = |
3514 |
SDValue Clz = |
| 3515 |
SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); |
3515 |
SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); |
| 3516 |
SDValue ShiftOps[] = |
3516 |
SDValue ShiftOps[] = |
| 3517 |
{ Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), S->getI32Imm(31, dl) }; |
3517 |
{ Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), S->getI32Imm(31, dl) }; |
| 3518 |
SDValue Shift = |
3518 |
SDValue Shift = |
| 3519 |
SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0); |
3519 |
SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0); |
| 3520 |
SDValue Xori = |
3520 |
SDValue Xori = |
| 3521 |
SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift, |
3521 |
SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift, |
| 3522 |
S->getI32Imm(1, dl)), 0); |
3522 |
S->getI32Imm(1, dl)), 0); |
| 3523 |
return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Xori), 0); |
3523 |
return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Xori), 0); |
| 3524 |
} |
3524 |
} |
| 3525 |
case ISD::SETGE: { |
3525 |
case ISD::SETGE: { |
| 3526 |
// (sext (setcc %a, %b, setge)) -> (add (lshr (sub %a, %b), 63), -1) |
3526 |
// (sext (setcc %a, %b, setge)) -> (add (lshr (sub %a, %b), 63), -1) |
| 3527 |
// (sext (setcc %a, 0, setge)) -> (ashr (~ %a), 31) |
3527 |
// (sext (setcc %a, 0, setge)) -> (ashr (~ %a), 31) |
| 3528 |
if (IsRHSZero) |
3528 |
if (IsRHSZero) |
| 3529 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); |
3529 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); |
| 3530 |
|
3530 |
|
| 3531 |
// Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a) |
3531 |
// Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a) |
| 3532 |
// by swapping inputs and falling through. |
3532 |
// by swapping inputs and falling through. |
| 3533 |
std::swap(LHS, RHS); |
3533 |
std::swap(LHS, RHS); |
| 3534 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
3534 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
| 3535 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
3535 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
| 3536 |
[[fallthrough]]; |
3536 |
[[fallthrough]]; |
| 3537 |
} |
3537 |
} |
| 3538 |
case ISD::SETLE: { |
3538 |
case ISD::SETLE: { |
| 3539 |
if (CmpInGPR == ICGPR_NonExtIn) |
3539 |
if (CmpInGPR == ICGPR_NonExtIn) |
| 3540 |
return SDValue(); |
3540 |
return SDValue(); |
| 3541 |
// (sext (setcc %a, %b, setge)) -> (add (lshr (sub %b, %a), 63), -1) |
3541 |
// (sext (setcc %a, %b, setge)) -> (add (lshr (sub %b, %a), 63), -1) |
| 3542 |
// (sext (setcc %a, 0, setle)) -> (add (lshr (- %a), 63), -1) |
3542 |
// (sext (setcc %a, 0, setle)) -> (add (lshr (- %a), 63), -1) |
| 3543 |
if (IsRHSZero) |
3543 |
if (IsRHSZero) |
| 3544 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); |
3544 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); |
| 3545 |
|
3545 |
|
| 3546 |
// The upper 32-bits of the register can't be undefined for this sequence. |
3546 |
// The upper 32-bits of the register can't be undefined for this sequence. |
| 3547 |
LHS = signExtendInputIfNeeded(LHS); |
3547 |
LHS = signExtendInputIfNeeded(LHS); |
| 3548 |
RHS = signExtendInputIfNeeded(RHS); |
3548 |
RHS = signExtendInputIfNeeded(RHS); |
| 3549 |
SDValue SUBFNode = |
3549 |
SDValue SUBFNode = |
| 3550 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, MVT::Glue, |
3550 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, MVT::Glue, |
| 3551 |
LHS, RHS), 0); |
3551 |
LHS, RHS), 0); |
| 3552 |
SDValue Srdi = |
3552 |
SDValue Srdi = |
| 3553 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3553 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
| 3554 |
SUBFNode, S->getI64Imm(1, dl), |
3554 |
SUBFNode, S->getI64Imm(1, dl), |
| 3555 |
S->getI64Imm(63, dl)), 0); |
3555 |
S->getI64Imm(63, dl)), 0); |
| 3556 |
return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Srdi, |
3556 |
return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Srdi, |
| 3557 |
S->getI32Imm(-1, dl)), 0); |
3557 |
S->getI32Imm(-1, dl)), 0); |
| 3558 |
} |
3558 |
} |
| 3559 |
case ISD::SETGT: { |
3559 |
case ISD::SETGT: { |
| 3560 |
// (sext (setcc %a, %b, setgt)) -> (ashr (sub %b, %a), 63) |
3560 |
// (sext (setcc %a, %b, setgt)) -> (ashr (sub %b, %a), 63) |
| 3561 |
// (sext (setcc %a, -1, setgt)) -> (ashr (~ %a), 31) |
3561 |
// (sext (setcc %a, -1, setgt)) -> (ashr (~ %a), 31) |
| 3562 |
// (sext (setcc %a, 0, setgt)) -> (ashr (- %a), 63) |
3562 |
// (sext (setcc %a, 0, setgt)) -> (ashr (- %a), 63) |
| 3563 |
if (IsRHSNegOne) |
3563 |
if (IsRHSNegOne) |
| 3564 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); |
3564 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); |
| 3565 |
if (IsRHSZero) { |
3565 |
if (IsRHSZero) { |
| 3566 |
if (CmpInGPR == ICGPR_NonExtIn) |
3566 |
if (CmpInGPR == ICGPR_NonExtIn) |
| 3567 |
return SDValue(); |
3567 |
return SDValue(); |
| 3568 |
// The upper 32-bits of the register can't be undefined for this sequence. |
3568 |
// The upper 32-bits of the register can't be undefined for this sequence. |
| 3569 |
LHS = signExtendInputIfNeeded(LHS); |
3569 |
LHS = signExtendInputIfNeeded(LHS); |
| 3570 |
RHS = signExtendInputIfNeeded(RHS); |
3570 |
RHS = signExtendInputIfNeeded(RHS); |
| 3571 |
SDValue Neg = |
3571 |
SDValue Neg = |
| 3572 |
SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); |
3572 |
SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); |
| 3573 |
return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Neg, |
3573 |
return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Neg, |
| 3574 |
S->getI64Imm(63, dl)), 0); |
3574 |
S->getI64Imm(63, dl)), 0); |
| 3575 |
} |
3575 |
} |
| 3576 |
// Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as |
3576 |
// Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as |
| 3577 |
// (%b < %a) by swapping inputs and falling through. |
3577 |
// (%b < %a) by swapping inputs and falling through. |
| 3578 |
std::swap(LHS, RHS); |
3578 |
std::swap(LHS, RHS); |
| 3579 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
3579 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
| 3580 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
3580 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
| 3581 |
IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; |
3581 |
IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; |
| 3582 |
[[fallthrough]]; |
3582 |
[[fallthrough]]; |
| 3583 |
} |
3583 |
} |
| 3584 |
case ISD::SETLT: { |
3584 |
case ISD::SETLT: { |
| 3585 |
// (sext (setcc %a, %b, setgt)) -> (ashr (sub %a, %b), 63) |
3585 |
// (sext (setcc %a, %b, setgt)) -> (ashr (sub %a, %b), 63) |
| 3586 |
// (sext (setcc %a, 1, setgt)) -> (add (lshr (- %a), 63), -1) |
3586 |
// (sext (setcc %a, 1, setgt)) -> (add (lshr (- %a), 63), -1) |
| 3587 |
// (sext (setcc %a, 0, setgt)) -> (ashr %a, 31) |
3587 |
// (sext (setcc %a, 0, setgt)) -> (ashr %a, 31) |
| 3588 |
if (IsRHSOne) { |
3588 |
if (IsRHSOne) { |
| 3589 |
if (CmpInGPR == ICGPR_NonExtIn) |
3589 |
if (CmpInGPR == ICGPR_NonExtIn) |
| 3590 |
return SDValue(); |
3590 |
return SDValue(); |
| 3591 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); |
3591 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); |
| 3592 |
} |
3592 |
} |
| 3593 |
if (IsRHSZero) |
3593 |
if (IsRHSZero) |
| 3594 |
return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, LHS, |
3594 |
return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, LHS, |
| 3595 |
S->getI32Imm(31, dl)), 0); |
3595 |
S->getI32Imm(31, dl)), 0); |
| 3596 |
|
3596 |
|
| 3597 |
if (CmpInGPR == ICGPR_NonExtIn) |
3597 |
if (CmpInGPR == ICGPR_NonExtIn) |
| 3598 |
return SDValue(); |
3598 |
return SDValue(); |
| 3599 |
// The upper 32-bits of the register can't be undefined for this sequence. |
3599 |
// The upper 32-bits of the register can't be undefined for this sequence. |
| 3600 |
LHS = signExtendInputIfNeeded(LHS); |
3600 |
LHS = signExtendInputIfNeeded(LHS); |
| 3601 |
RHS = signExtendInputIfNeeded(RHS); |
3601 |
RHS = signExtendInputIfNeeded(RHS); |
| 3602 |
SDValue SUBFNode = |
3602 |
SDValue SUBFNode = |
| 3603 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); |
3603 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); |
| 3604 |
return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, |
3604 |
return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, |
| 3605 |
SUBFNode, S->getI64Imm(63, dl)), 0); |
3605 |
SUBFNode, S->getI64Imm(63, dl)), 0); |
| 3606 |
} |
3606 |
} |
| 3607 |
case ISD::SETUGE: |
3607 |
case ISD::SETUGE: |
| 3608 |
// (sext (setcc %a, %b, setuge)) -> (add (lshr (sub %a, %b), 63), -1) |
3608 |
// (sext (setcc %a, %b, setuge)) -> (add (lshr (sub %a, %b), 63), -1) |
| 3609 |
// (sext (setcc %a, %b, setule)) -> (add (lshr (sub %b, %a), 63), -1) |
3609 |
// (sext (setcc %a, %b, setule)) -> (add (lshr (sub %b, %a), 63), -1) |
| 3610 |
std::swap(LHS, RHS); |
3610 |
std::swap(LHS, RHS); |
| 3611 |
[[fallthrough]]; |
3611 |
[[fallthrough]]; |
| 3612 |
case ISD::SETULE: { |
3612 |
case ISD::SETULE: { |
| 3613 |
if (CmpInGPR == ICGPR_NonExtIn) |
3613 |
if (CmpInGPR == ICGPR_NonExtIn) |
| 3614 |
return SDValue(); |
3614 |
return SDValue(); |
| 3615 |
// The upper 32-bits of the register can't be undefined for this sequence. |
3615 |
// The upper 32-bits of the register can't be undefined for this sequence. |
| 3616 |
LHS = zeroExtendInputIfNeeded(LHS); |
3616 |
LHS = zeroExtendInputIfNeeded(LHS); |
| 3617 |
RHS = zeroExtendInputIfNeeded(RHS); |
3617 |
RHS = zeroExtendInputIfNeeded(RHS); |
| 3618 |
SDValue Subtract = |
3618 |
SDValue Subtract = |
| 3619 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); |
3619 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); |
| 3620 |
SDValue Shift = |
3620 |
SDValue Shift = |
| 3621 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Subtract, |
3621 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Subtract, |
| 3622 |
S->getI32Imm(1, dl), S->getI32Imm(63,dl)), |
3622 |
S->getI32Imm(1, dl), S->getI32Imm(63,dl)), |
| 3623 |
0); |
3623 |
0); |
| 3624 |
return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Shift, |
3624 |
return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Shift, |
| 3625 |
S->getI32Imm(-1, dl)), 0); |
3625 |
S->getI32Imm(-1, dl)), 0); |
| 3626 |
} |
3626 |
} |
| 3627 |
case ISD::SETUGT: |
3627 |
case ISD::SETUGT: |
| 3628 |
// (sext (setcc %a, %b, setugt)) -> (ashr (sub %b, %a), 63) |
3628 |
// (sext (setcc %a, %b, setugt)) -> (ashr (sub %b, %a), 63) |
| 3629 |
// (sext (setcc %a, %b, setugt)) -> (ashr (sub %a, %b), 63) |
3629 |
// (sext (setcc %a, %b, setugt)) -> (ashr (sub %a, %b), 63) |
| 3630 |
std::swap(LHS, RHS); |
3630 |
std::swap(LHS, RHS); |
| 3631 |
[[fallthrough]]; |
3631 |
[[fallthrough]]; |
| 3632 |
case ISD::SETULT: { |
3632 |
case ISD::SETULT: { |
| 3633 |
if (CmpInGPR == ICGPR_NonExtIn) |
3633 |
if (CmpInGPR == ICGPR_NonExtIn) |
| 3634 |
return SDValue(); |
3634 |
return SDValue(); |
| 3635 |
// The upper 32-bits of the register can't be undefined for this sequence. |
3635 |
// The upper 32-bits of the register can't be undefined for this sequence. |
| 3636 |
LHS = zeroExtendInputIfNeeded(LHS); |
3636 |
LHS = zeroExtendInputIfNeeded(LHS); |
| 3637 |
RHS = zeroExtendInputIfNeeded(RHS); |
3637 |
RHS = zeroExtendInputIfNeeded(RHS); |
| 3638 |
SDValue Subtract = |
3638 |
SDValue Subtract = |
| 3639 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); |
3639 |
SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); |
| 3640 |
return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, |
3640 |
return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, |
| 3641 |
Subtract, S->getI64Imm(63, dl)), 0); |
3641 |
Subtract, S->getI64Imm(63, dl)), 0); |
| 3642 |
} |
3642 |
} |
| 3643 |
} |
3643 |
} |
| 3644 |
} |
3644 |
} |
| 3645 |
|
3645 |
|
| 3646 |
/// Produces a zero-extended result of comparing two 64-bit values according to |
3646 |
/// Produces a zero-extended result of comparing two 64-bit values according to |
| 3647 |
/// the passed condition code. |
3647 |
/// the passed condition code. |
| 3648 |
SDValue |
3648 |
SDValue |
| 3649 |
IntegerCompareEliminator::get64BitZExtCompare(SDValue LHS, SDValue RHS, |
3649 |
IntegerCompareEliminator::get64BitZExtCompare(SDValue LHS, SDValue RHS, |
| 3650 |
ISD::CondCode CC, |
3650 |
ISD::CondCode CC, |
| 3651 |
int64_t RHSValue, SDLoc dl) { |
3651 |
int64_t RHSValue, SDLoc dl) { |
| 3652 |
if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 || |
3652 |
if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 || |
| 3653 |
CmpInGPR == ICGPR_ZextI32 || CmpInGPR == ICGPR_Sext) |
3653 |
CmpInGPR == ICGPR_ZextI32 || CmpInGPR == ICGPR_Sext) |
| 3654 |
return SDValue(); |
3654 |
return SDValue(); |
| 3655 |
bool IsRHSZero = RHSValue == 0; |
3655 |
bool IsRHSZero = RHSValue == 0; |
| 3656 |
bool IsRHSOne = RHSValue == 1; |
3656 |
bool IsRHSOne = RHSValue == 1; |
| 3657 |
bool IsRHSNegOne = RHSValue == -1LL; |
3657 |
bool IsRHSNegOne = RHSValue == -1LL; |
| 3658 |
switch (CC) { |
3658 |
switch (CC) { |
| 3659 |
default: return SDValue(); |
3659 |
default: return SDValue(); |
| 3660 |
case ISD::SETEQ: { |
3660 |
case ISD::SETEQ: { |
| 3661 |
// (zext (setcc %a, %b, seteq)) -> (lshr (ctlz (xor %a, %b)), 6) |
3661 |
// (zext (setcc %a, %b, seteq)) -> (lshr (ctlz (xor %a, %b)), 6) |
| 3662 |
// (zext (setcc %a, 0, seteq)) -> (lshr (ctlz %a), 6) |
3662 |
// (zext (setcc %a, 0, seteq)) -> (lshr (ctlz %a), 6) |
| 3663 |
SDValue Xor = IsRHSZero ? LHS : |
3663 |
SDValue Xor = IsRHSZero ? LHS : |
| 3664 |
SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); |
3664 |
SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); |
| 3665 |
SDValue Clz = |
3665 |
SDValue Clz = |
| 3666 |
SDValue(CurDAG->getMachineNode(PPC::CNTLZD, dl, MVT::i64, Xor), 0); |
3666 |
SDValue(CurDAG->getMachineNode(PPC::CNTLZD, dl, MVT::i64, Xor), 0); |
| 3667 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Clz, |
3667 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Clz, |
| 3668 |
S->getI64Imm(58, dl), |
3668 |
S->getI64Imm(58, dl), |
| 3669 |
S->getI64Imm(63, dl)), 0); |
3669 |
S->getI64Imm(63, dl)), 0); |
| 3670 |
} |
3670 |
} |
| 3671 |
case ISD::SETNE: { |
3671 |
case ISD::SETNE: { |
| 3672 |
// {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1) |
3672 |
// {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1) |
| 3673 |
// (zext (setcc %a, %b, setne)) -> (sube addc.reg, addc.reg, addc.CA) |
3673 |
// (zext (setcc %a, %b, setne)) -> (sube addc.reg, addc.reg, addc.CA) |
| 3674 |
// {addcz.reg, addcz.CA} = (addcarry %a, -1) |
3674 |
// {addcz.reg, addcz.CA} = (addcarry %a, -1) |
| 3675 |
// (zext (setcc %a, 0, setne)) -> (sube addcz.reg, addcz.reg, addcz.CA) |
3675 |
// (zext (setcc %a, 0, setne)) -> (sube addcz.reg, addcz.reg, addcz.CA) |
| 3676 |
SDValue Xor = IsRHSZero ? LHS : |
3676 |
SDValue Xor = IsRHSZero ? LHS : |
| 3677 |
SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); |
3677 |
SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); |
| 3678 |
SDValue AC = |
3678 |
SDValue AC = |
| 3679 |
SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue, |
3679 |
SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue, |
| 3680 |
Xor, S->getI32Imm(~0U, dl)), 0); |
3680 |
Xor, S->getI32Imm(~0U, dl)), 0); |
| 3681 |
return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, AC, |
3681 |
return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, AC, |
| 3682 |
Xor, AC.getValue(1)), 0); |
3682 |
Xor, AC.getValue(1)), 0); |
| 3683 |
} |
3683 |
} |
| 3684 |
case ISD::SETGE: { |
3684 |
case ISD::SETGE: { |
| 3685 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
3685 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
| 3686 |
// (zext (setcc %a, %b, setge)) -> |
3686 |
// (zext (setcc %a, %b, setge)) -> |
| 3687 |
// (adde (lshr %b, 63), (ashr %a, 63), subc.CA) |
3687 |
// (adde (lshr %b, 63), (ashr %a, 63), subc.CA) |
| 3688 |
// (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 63) |
3688 |
// (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 63) |
| 3689 |
if (IsRHSZero) |
3689 |
if (IsRHSZero) |
| 3690 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); |
3690 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); |
| 3691 |
std::swap(LHS, RHS); |
3691 |
std::swap(LHS, RHS); |
| 3692 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
3692 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
| 3693 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
3693 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
| 3694 |
[[fallthrough]]; |
3694 |
[[fallthrough]]; |
| 3695 |
} |
3695 |
} |
| 3696 |
case ISD::SETLE: { |
3696 |
case ISD::SETLE: { |
| 3697 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
3697 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
| 3698 |
// (zext (setcc %a, %b, setge)) -> |
3698 |
// (zext (setcc %a, %b, setge)) -> |
| 3699 |
// (adde (lshr %a, 63), (ashr %b, 63), subc.CA) |
3699 |
// (adde (lshr %a, 63), (ashr %b, 63), subc.CA) |
| 3700 |
// (zext (setcc %a, 0, setge)) -> (lshr (or %a, (add %a, -1)), 63) |
3700 |
// (zext (setcc %a, 0, setge)) -> (lshr (or %a, (add %a, -1)), 63) |
| 3701 |
if (IsRHSZero) |
3701 |
if (IsRHSZero) |
| 3702 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); |
3702 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); |
| 3703 |
SDValue ShiftL = |
3703 |
SDValue ShiftL = |
| 3704 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, |
3704 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, |
| 3705 |
S->getI64Imm(1, dl), |
3705 |
S->getI64Imm(1, dl), |
| 3706 |
S->getI64Imm(63, dl)), 0); |
3706 |
S->getI64Imm(63, dl)), 0); |
| 3707 |
SDValue ShiftR = |
3707 |
SDValue ShiftR = |
| 3708 |
SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS, |
3708 |
SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS, |
| 3709 |
S->getI64Imm(63, dl)), 0); |
3709 |
S->getI64Imm(63, dl)), 0); |
| 3710 |
SDValue SubtractCarry = |
3710 |
SDValue SubtractCarry = |
| 3711 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3711 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
| 3712 |
LHS, RHS), 1); |
3712 |
LHS, RHS), 1); |
| 3713 |
return SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, |
3713 |
return SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, |
| 3714 |
ShiftR, ShiftL, SubtractCarry), 0); |
3714 |
ShiftR, ShiftL, SubtractCarry), 0); |
| 3715 |
} |
3715 |
} |
| 3716 |
case ISD::SETGT: { |
3716 |
case ISD::SETGT: { |
| 3717 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
3717 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
| 3718 |
// (zext (setcc %a, %b, setgt)) -> |
3718 |
// (zext (setcc %a, %b, setgt)) -> |
| 3719 |
// (xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1) |
3719 |
// (xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1) |
| 3720 |
// (zext (setcc %a, 0, setgt)) -> (lshr (nor (add %a, -1), %a), 63) |
3720 |
// (zext (setcc %a, 0, setgt)) -> (lshr (nor (add %a, -1), %a), 63) |
| 3721 |
if (IsRHSNegOne) |
3721 |
if (IsRHSNegOne) |
| 3722 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); |
3722 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); |
| 3723 |
if (IsRHSZero) { |
3723 |
if (IsRHSZero) { |
| 3724 |
SDValue Addi = |
3724 |
SDValue Addi = |
| 3725 |
SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, |
3725 |
SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, |
| 3726 |
S->getI64Imm(~0ULL, dl)), 0); |
3726 |
S->getI64Imm(~0ULL, dl)), 0); |
| 3727 |
SDValue Nor = |
3727 |
SDValue Nor = |
| 3728 |
SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Addi, LHS), 0); |
3728 |
SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Addi, LHS), 0); |
| 3729 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Nor, |
3729 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Nor, |
| 3730 |
S->getI64Imm(1, dl), |
3730 |
S->getI64Imm(1, dl), |
| 3731 |
S->getI64Imm(63, dl)), 0); |
3731 |
S->getI64Imm(63, dl)), 0); |
| 3732 |
} |
3732 |
} |
| 3733 |
std::swap(LHS, RHS); |
3733 |
std::swap(LHS, RHS); |
| 3734 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
3734 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
| 3735 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
3735 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
| 3736 |
IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; |
3736 |
IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; |
| 3737 |
[[fallthrough]]; |
3737 |
[[fallthrough]]; |
| 3738 |
} |
3738 |
} |
| 3739 |
case ISD::SETLT: { |
3739 |
case ISD::SETLT: { |
| 3740 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
3740 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
| 3741 |
// (zext (setcc %a, %b, setlt)) -> |
3741 |
// (zext (setcc %a, %b, setlt)) -> |
| 3742 |
// (xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1) |
3742 |
// (xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1) |
| 3743 |
// (zext (setcc %a, 0, setlt)) -> (lshr %a, 63) |
3743 |
// (zext (setcc %a, 0, setlt)) -> (lshr %a, 63) |
| 3744 |
if (IsRHSOne) |
3744 |
if (IsRHSOne) |
| 3745 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); |
3745 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); |
| 3746 |
if (IsRHSZero) |
3746 |
if (IsRHSZero) |
| 3747 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, |
3747 |
return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, |
| 3748 |
S->getI64Imm(1, dl), |
3748 |
S->getI64Imm(1, dl), |
| 3749 |
S->getI64Imm(63, dl)), 0); |
3749 |
S->getI64Imm(63, dl)), 0); |
| 3750 |
SDValue SRADINode = |
3750 |
SDValue SRADINode = |
| 3751 |
SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, |
3751 |
SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, |
| 3752 |
LHS, S->getI64Imm(63, dl)), 0); |
3752 |
LHS, S->getI64Imm(63, dl)), 0); |
| 3753 |
SDValue SRDINode = |
3753 |
SDValue SRDINode = |
| 3754 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3754 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
| 3755 |
RHS, S->getI64Imm(1, dl), |
3755 |
RHS, S->getI64Imm(1, dl), |
| 3756 |
S->getI64Imm(63, dl)), 0); |
3756 |
S->getI64Imm(63, dl)), 0); |
| 3757 |
SDValue SUBFC8Carry = |
3757 |
SDValue SUBFC8Carry = |
| 3758 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3758 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
| 3759 |
RHS, LHS), 1); |
3759 |
RHS, LHS), 1); |
| 3760 |
SDValue ADDE8Node = |
3760 |
SDValue ADDE8Node = |
| 3761 |
SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, |
3761 |
SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, |
| 3762 |
SRDINode, SRADINode, SUBFC8Carry), 0); |
3762 |
SRDINode, SRADINode, SUBFC8Carry), 0); |
| 3763 |
return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, |
3763 |
return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, |
| 3764 |
ADDE8Node, S->getI64Imm(1, dl)), 0); |
3764 |
ADDE8Node, S->getI64Imm(1, dl)), 0); |
| 3765 |
} |
3765 |
} |
| 3766 |
case ISD::SETUGE: |
3766 |
case ISD::SETUGE: |
| 3767 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
3767 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
| 3768 |
// (zext (setcc %a, %b, setuge)) -> (add (sube %b, %b, subc.CA), 1) |
3768 |
// (zext (setcc %a, %b, setuge)) -> (add (sube %b, %b, subc.CA), 1) |
| 3769 |
std::swap(LHS, RHS); |
3769 |
std::swap(LHS, RHS); |
| 3770 |
[[fallthrough]]; |
3770 |
[[fallthrough]]; |
| 3771 |
case ISD::SETULE: { |
3771 |
case ISD::SETULE: { |
| 3772 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
3772 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
| 3773 |
// (zext (setcc %a, %b, setule)) -> (add (sube %a, %a, subc.CA), 1) |
3773 |
// (zext (setcc %a, %b, setule)) -> (add (sube %a, %a, subc.CA), 1) |
| 3774 |
SDValue SUBFC8Carry = |
3774 |
SDValue SUBFC8Carry = |
| 3775 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3775 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
| 3776 |
LHS, RHS), 1); |
3776 |
LHS, RHS), 1); |
| 3777 |
SDValue SUBFE8Node = |
3777 |
SDValue SUBFE8Node = |
| 3778 |
SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, |
3778 |
SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, |
| 3779 |
LHS, LHS, SUBFC8Carry), 0); |
3779 |
LHS, LHS, SUBFC8Carry), 0); |
| 3780 |
return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, |
3780 |
return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, |
| 3781 |
SUBFE8Node, S->getI64Imm(1, dl)), 0); |
3781 |
SUBFE8Node, S->getI64Imm(1, dl)), 0); |
| 3782 |
} |
3782 |
} |
| 3783 |
case ISD::SETUGT: |
3783 |
case ISD::SETUGT: |
| 3784 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
3784 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
| 3785 |
// (zext (setcc %a, %b, setugt)) -> -(sube %b, %b, subc.CA) |
3785 |
// (zext (setcc %a, %b, setugt)) -> -(sube %b, %b, subc.CA) |
| 3786 |
std::swap(LHS, RHS); |
3786 |
std::swap(LHS, RHS); |
| 3787 |
[[fallthrough]]; |
3787 |
[[fallthrough]]; |
| 3788 |
case ISD::SETULT: { |
3788 |
case ISD::SETULT: { |
| 3789 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
3789 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
| 3790 |
// (zext (setcc %a, %b, setult)) -> -(sube %a, %a, subc.CA) |
3790 |
// (zext (setcc %a, %b, setult)) -> -(sube %a, %a, subc.CA) |
| 3791 |
SDValue SubtractCarry = |
3791 |
SDValue SubtractCarry = |
| 3792 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3792 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
| 3793 |
RHS, LHS), 1); |
3793 |
RHS, LHS), 1); |
| 3794 |
SDValue ExtSub = |
3794 |
SDValue ExtSub = |
| 3795 |
SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, |
3795 |
SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, |
| 3796 |
LHS, LHS, SubtractCarry), 0); |
3796 |
LHS, LHS, SubtractCarry), 0); |
| 3797 |
return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, |
3797 |
return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, |
| 3798 |
ExtSub), 0); |
3798 |
ExtSub), 0); |
| 3799 |
} |
3799 |
} |
| 3800 |
} |
3800 |
} |
| 3801 |
} |
3801 |
} |
| 3802 |
|
3802 |
|
| 3803 |
/// Produces a sign-extended result of comparing two 64-bit values according to |
3803 |
/// Produces a sign-extended result of comparing two 64-bit values according to |
| 3804 |
/// the passed condition code. |
3804 |
/// the passed condition code. |
| 3805 |
SDValue |
3805 |
SDValue |
| 3806 |
IntegerCompareEliminator::get64BitSExtCompare(SDValue LHS, SDValue RHS, |
3806 |
IntegerCompareEliminator::get64BitSExtCompare(SDValue LHS, SDValue RHS, |
| 3807 |
ISD::CondCode CC, |
3807 |
ISD::CondCode CC, |
| 3808 |
int64_t RHSValue, SDLoc dl) { |
3808 |
int64_t RHSValue, SDLoc dl) { |
| 3809 |
if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 || |
3809 |
if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 || |
| 3810 |
CmpInGPR == ICGPR_ZextI32 || CmpInGPR == ICGPR_Zext) |
3810 |
CmpInGPR == ICGPR_ZextI32 || CmpInGPR == ICGPR_Zext) |
| 3811 |
return SDValue(); |
3811 |
return SDValue(); |
| 3812 |
bool IsRHSZero = RHSValue == 0; |
3812 |
bool IsRHSZero = RHSValue == 0; |
| 3813 |
bool IsRHSOne = RHSValue == 1; |
3813 |
bool IsRHSOne = RHSValue == 1; |
| 3814 |
bool IsRHSNegOne = RHSValue == -1LL; |
3814 |
bool IsRHSNegOne = RHSValue == -1LL; |
| 3815 |
switch (CC) { |
3815 |
switch (CC) { |
| 3816 |
default: return SDValue(); |
3816 |
default: return SDValue(); |
| 3817 |
case ISD::SETEQ: { |
3817 |
case ISD::SETEQ: { |
| 3818 |
// {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1) |
3818 |
// {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1) |
| 3819 |
// (sext (setcc %a, %b, seteq)) -> (sube addc.reg, addc.reg, addc.CA) |
3819 |
// (sext (setcc %a, %b, seteq)) -> (sube addc.reg, addc.reg, addc.CA) |
| 3820 |
// {addcz.reg, addcz.CA} = (addcarry %a, -1) |
3820 |
// {addcz.reg, addcz.CA} = (addcarry %a, -1) |
| 3821 |
// (sext (setcc %a, 0, seteq)) -> (sube addcz.reg, addcz.reg, addcz.CA) |
3821 |
// (sext (setcc %a, 0, seteq)) -> (sube addcz.reg, addcz.reg, addcz.CA) |
| 3822 |
SDValue AddInput = IsRHSZero ? LHS : |
3822 |
SDValue AddInput = IsRHSZero ? LHS : |
| 3823 |
SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); |
3823 |
SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); |
| 3824 |
SDValue Addic = |
3824 |
SDValue Addic = |
| 3825 |
SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue, |
3825 |
SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue, |
| 3826 |
AddInput, S->getI32Imm(~0U, dl)), 0); |
3826 |
AddInput, S->getI32Imm(~0U, dl)), 0); |
| 3827 |
return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, Addic, |
3827 |
return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, Addic, |
| 3828 |
Addic, Addic.getValue(1)), 0); |
3828 |
Addic, Addic.getValue(1)), 0); |
| 3829 |
} |
3829 |
} |
| 3830 |
case ISD::SETNE: { |
3830 |
case ISD::SETNE: { |
| 3831 |
// {subfc.reg, subfc.CA} = (subcarry 0, (xor %a, %b)) |
3831 |
// {subfc.reg, subfc.CA} = (subcarry 0, (xor %a, %b)) |
| 3832 |
// (sext (setcc %a, %b, setne)) -> (sube subfc.reg, subfc.reg, subfc.CA) |
3832 |
// (sext (setcc %a, %b, setne)) -> (sube subfc.reg, subfc.reg, subfc.CA) |
| 3833 |
// {subfcz.reg, subfcz.CA} = (subcarry 0, %a) |
3833 |
// {subfcz.reg, subfcz.CA} = (subcarry 0, %a) |
| 3834 |
// (sext (setcc %a, 0, setne)) -> (sube subfcz.reg, subfcz.reg, subfcz.CA) |
3834 |
// (sext (setcc %a, 0, setne)) -> (sube subfcz.reg, subfcz.reg, subfcz.CA) |
| 3835 |
SDValue Xor = IsRHSZero ? LHS : |
3835 |
SDValue Xor = IsRHSZero ? LHS : |
| 3836 |
SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); |
3836 |
SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); |
| 3837 |
SDValue SC = |
3837 |
SDValue SC = |
| 3838 |
SDValue(CurDAG->getMachineNode(PPC::SUBFIC8, dl, MVT::i64, MVT::Glue, |
3838 |
SDValue(CurDAG->getMachineNode(PPC::SUBFIC8, dl, MVT::i64, MVT::Glue, |
| 3839 |
Xor, S->getI32Imm(0, dl)), 0); |
3839 |
Xor, S->getI32Imm(0, dl)), 0); |
| 3840 |
return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, SC, |
3840 |
return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, SC, |
| 3841 |
SC, SC.getValue(1)), 0); |
3841 |
SC, SC.getValue(1)), 0); |
| 3842 |
} |
3842 |
} |
| 3843 |
case ISD::SETGE: { |
3843 |
case ISD::SETGE: { |
| 3844 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
3844 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
| 3845 |
// (zext (setcc %a, %b, setge)) -> |
3845 |
// (zext (setcc %a, %b, setge)) -> |
| 3846 |
// (- (adde (lshr %b, 63), (ashr %a, 63), subc.CA)) |
3846 |
// (- (adde (lshr %b, 63), (ashr %a, 63), subc.CA)) |
| 3847 |
// (zext (setcc %a, 0, setge)) -> (~ (ashr %a, 63)) |
3847 |
// (zext (setcc %a, 0, setge)) -> (~ (ashr %a, 63)) |
| 3848 |
if (IsRHSZero) |
3848 |
if (IsRHSZero) |
| 3849 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); |
3849 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); |
| 3850 |
std::swap(LHS, RHS); |
3850 |
std::swap(LHS, RHS); |
| 3851 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
3851 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
| 3852 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
3852 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
| 3853 |
[[fallthrough]]; |
3853 |
[[fallthrough]]; |
| 3854 |
} |
3854 |
} |
| 3855 |
case ISD::SETLE: { |
3855 |
case ISD::SETLE: { |
| 3856 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
3856 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
| 3857 |
// (zext (setcc %a, %b, setge)) -> |
3857 |
// (zext (setcc %a, %b, setge)) -> |
| 3858 |
// (- (adde (lshr %a, 63), (ashr %b, 63), subc.CA)) |
3858 |
// (- (adde (lshr %a, 63), (ashr %b, 63), subc.CA)) |
| 3859 |
// (zext (setcc %a, 0, setge)) -> (ashr (or %a, (add %a, -1)), 63) |
3859 |
// (zext (setcc %a, 0, setge)) -> (ashr (or %a, (add %a, -1)), 63) |
| 3860 |
if (IsRHSZero) |
3860 |
if (IsRHSZero) |
| 3861 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); |
3861 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); |
| 3862 |
SDValue ShiftR = |
3862 |
SDValue ShiftR = |
| 3863 |
SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS, |
3863 |
SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS, |
| 3864 |
S->getI64Imm(63, dl)), 0); |
3864 |
S->getI64Imm(63, dl)), 0); |
| 3865 |
SDValue ShiftL = |
3865 |
SDValue ShiftL = |
| 3866 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, |
3866 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, |
| 3867 |
S->getI64Imm(1, dl), |
3867 |
S->getI64Imm(1, dl), |
| 3868 |
S->getI64Imm(63, dl)), 0); |
3868 |
S->getI64Imm(63, dl)), 0); |
| 3869 |
SDValue SubtractCarry = |
3869 |
SDValue SubtractCarry = |
| 3870 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3870 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
| 3871 |
LHS, RHS), 1); |
3871 |
LHS, RHS), 1); |
| 3872 |
SDValue Adde = |
3872 |
SDValue Adde = |
| 3873 |
SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, |
3873 |
SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, |
| 3874 |
ShiftR, ShiftL, SubtractCarry), 0); |
3874 |
ShiftR, ShiftL, SubtractCarry), 0); |
| 3875 |
return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, Adde), 0); |
3875 |
return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, Adde), 0); |
| 3876 |
} |
3876 |
} |
| 3877 |
case ISD::SETGT: { |
3877 |
case ISD::SETGT: { |
| 3878 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
3878 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
| 3879 |
// (zext (setcc %a, %b, setgt)) -> |
3879 |
// (zext (setcc %a, %b, setgt)) -> |
| 3880 |
// -(xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1) |
3880 |
// -(xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1) |
| 3881 |
// (zext (setcc %a, 0, setgt)) -> (ashr (nor (add %a, -1), %a), 63) |
3881 |
// (zext (setcc %a, 0, setgt)) -> (ashr (nor (add %a, -1), %a), 63) |
| 3882 |
if (IsRHSNegOne) |
3882 |
if (IsRHSNegOne) |
| 3883 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); |
3883 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); |
| 3884 |
if (IsRHSZero) { |
3884 |
if (IsRHSZero) { |
| 3885 |
SDValue Add = |
3885 |
SDValue Add = |
| 3886 |
SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, |
3886 |
SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, |
| 3887 |
S->getI64Imm(-1, dl)), 0); |
3887 |
S->getI64Imm(-1, dl)), 0); |
| 3888 |
SDValue Nor = |
3888 |
SDValue Nor = |
| 3889 |
SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Add, LHS), 0); |
3889 |
SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Add, LHS), 0); |
| 3890 |
return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Nor, |
3890 |
return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Nor, |
| 3891 |
S->getI64Imm(63, dl)), 0); |
3891 |
S->getI64Imm(63, dl)), 0); |
| 3892 |
} |
3892 |
} |
| 3893 |
std::swap(LHS, RHS); |
3893 |
std::swap(LHS, RHS); |
| 3894 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
3894 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
| 3895 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
3895 |
IsRHSZero = RHSConst && RHSConst->isZero(); |
| 3896 |
IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; |
3896 |
IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; |
| 3897 |
[[fallthrough]]; |
3897 |
[[fallthrough]]; |
| 3898 |
} |
3898 |
} |
| 3899 |
case ISD::SETLT: { |
3899 |
case ISD::SETLT: { |
| 3900 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
3900 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
| 3901 |
// (zext (setcc %a, %b, setlt)) -> |
3901 |
// (zext (setcc %a, %b, setlt)) -> |
| 3902 |
// -(xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1) |
3902 |
// -(xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1) |
| 3903 |
// (zext (setcc %a, 0, setlt)) -> (ashr %a, 63) |
3903 |
// (zext (setcc %a, 0, setlt)) -> (ashr %a, 63) |
| 3904 |
if (IsRHSOne) |
3904 |
if (IsRHSOne) |
| 3905 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); |
3905 |
return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); |
| 3906 |
if (IsRHSZero) { |
3906 |
if (IsRHSZero) { |
| 3907 |
return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, LHS, |
3907 |
return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, LHS, |
| 3908 |
S->getI64Imm(63, dl)), 0); |
3908 |
S->getI64Imm(63, dl)), 0); |
| 3909 |
} |
3909 |
} |
| 3910 |
SDValue SRADINode = |
3910 |
SDValue SRADINode = |
| 3911 |
SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, |
3911 |
SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, |
| 3912 |
LHS, S->getI64Imm(63, dl)), 0); |
3912 |
LHS, S->getI64Imm(63, dl)), 0); |
| 3913 |
SDValue SRDINode = |
3913 |
SDValue SRDINode = |
| 3914 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3914 |
SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
| 3915 |
RHS, S->getI64Imm(1, dl), |
3915 |
RHS, S->getI64Imm(1, dl), |
| 3916 |
S->getI64Imm(63, dl)), 0); |
3916 |
S->getI64Imm(63, dl)), 0); |
| 3917 |
SDValue SUBFC8Carry = |
3917 |
SDValue SUBFC8Carry = |
| 3918 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3918 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
| 3919 |
RHS, LHS), 1); |
3919 |
RHS, LHS), 1); |
| 3920 |
SDValue ADDE8Node = |
3920 |
SDValue ADDE8Node = |
| 3921 |
SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, |
3921 |
SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, |
| 3922 |
SRDINode, SRADINode, SUBFC8Carry), 0); |
3922 |
SRDINode, SRADINode, SUBFC8Carry), 0); |
| 3923 |
SDValue XORI8Node = |
3923 |
SDValue XORI8Node = |
| 3924 |
SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, |
3924 |
SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, |
| 3925 |
ADDE8Node, S->getI64Imm(1, dl)), 0); |
3925 |
ADDE8Node, S->getI64Imm(1, dl)), 0); |
| 3926 |
return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, |
3926 |
return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, |
| 3927 |
XORI8Node), 0); |
3927 |
XORI8Node), 0); |
| 3928 |
} |
3928 |
} |
| 3929 |
case ISD::SETUGE: |
3929 |
case ISD::SETUGE: |
| 3930 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
3930 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
| 3931 |
// (sext (setcc %a, %b, setuge)) -> ~(sube %b, %b, subc.CA) |
3931 |
// (sext (setcc %a, %b, setuge)) -> ~(sube %b, %b, subc.CA) |
| 3932 |
std::swap(LHS, RHS); |
3932 |
std::swap(LHS, RHS); |
| 3933 |
[[fallthrough]]; |
3933 |
[[fallthrough]]; |
| 3934 |
case ISD::SETULE: { |
3934 |
case ISD::SETULE: { |
| 3935 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
3935 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
| 3936 |
// (sext (setcc %a, %b, setule)) -> ~(sube %a, %a, subc.CA) |
3936 |
// (sext (setcc %a, %b, setule)) -> ~(sube %a, %a, subc.CA) |
| 3937 |
SDValue SubtractCarry = |
3937 |
SDValue SubtractCarry = |
| 3938 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3938 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
| 3939 |
LHS, RHS), 1); |
3939 |
LHS, RHS), 1); |
| 3940 |
SDValue ExtSub = |
3940 |
SDValue ExtSub = |
| 3941 |
SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, LHS, |
3941 |
SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, LHS, |
| 3942 |
LHS, SubtractCarry), 0); |
3942 |
LHS, SubtractCarry), 0); |
| 3943 |
return SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, |
3943 |
return SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, |
| 3944 |
ExtSub, ExtSub), 0); |
3944 |
ExtSub, ExtSub), 0); |
| 3945 |
} |
3945 |
} |
| 3946 |
case ISD::SETUGT: |
3946 |
case ISD::SETUGT: |
| 3947 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
3947 |
// {subc.reg, subc.CA} = (subcarry %b, %a) |
| 3948 |
// (sext (setcc %a, %b, setugt)) -> (sube %b, %b, subc.CA) |
3948 |
// (sext (setcc %a, %b, setugt)) -> (sube %b, %b, subc.CA) |
| 3949 |
std::swap(LHS, RHS); |
3949 |
std::swap(LHS, RHS); |
| 3950 |
[[fallthrough]]; |
3950 |
[[fallthrough]]; |
| 3951 |
case ISD::SETULT: { |
3951 |
case ISD::SETULT: { |
| 3952 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
3952 |
// {subc.reg, subc.CA} = (subcarry %a, %b) |
| 3953 |
// (sext (setcc %a, %b, setult)) -> (sube %a, %a, subc.CA) |
3953 |
// (sext (setcc %a, %b, setult)) -> (sube %a, %a, subc.CA) |
| 3954 |
SDValue SubCarry = |
3954 |
SDValue SubCarry = |
| 3955 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3955 |
SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
| 3956 |
RHS, LHS), 1); |
3956 |
RHS, LHS), 1); |
| 3957 |
return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, |
3957 |
return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, |
| 3958 |
LHS, LHS, SubCarry), 0); |
3958 |
LHS, LHS, SubCarry), 0); |
| 3959 |
} |
3959 |
} |
| 3960 |
} |
3960 |
} |
| 3961 |
} |
3961 |
} |
| 3962 |
|
3962 |
|
| 3963 |
/// Do all uses of this SDValue need the result in a GPR? |
3963 |
/// Do all uses of this SDValue need the result in a GPR? |
| 3964 |
/// This is meant to be used on values that have type i1 since |
3964 |
/// This is meant to be used on values that have type i1 since |
| 3965 |
/// it is somewhat meaningless to ask if values of other types |
3965 |
/// it is somewhat meaningless to ask if values of other types |
| 3966 |
/// should be kept in GPR's. |
3966 |
/// should be kept in GPR's. |
| 3967 |
static bool allUsesExtend(SDValue Compare, SelectionDAG *CurDAG) { |
3967 |
static bool allUsesExtend(SDValue Compare, SelectionDAG *CurDAG) { |
| 3968 |
assert(Compare.getOpcode() == ISD::SETCC && |
3968 |
assert(Compare.getOpcode() == ISD::SETCC && |
| 3969 |
"An ISD::SETCC node required here."); |
3969 |
"An ISD::SETCC node required here."); |
| 3970 |
|
3970 |
|
| 3971 |
// For values that have a single use, the caller should obviously already have |
3971 |
// For values that have a single use, the caller should obviously already have |
| 3972 |
// checked if that use is an extending use. We check the other uses here. |
3972 |
// checked if that use is an extending use. We check the other uses here. |
| 3973 |
if (Compare.hasOneUse()) |
3973 |
if (Compare.hasOneUse()) |
| 3974 |
return true; |
3974 |
return true; |
| 3975 |
// We want the value in a GPR if it is being extended, used for a select, or |
3975 |
// We want the value in a GPR if it is being extended, used for a select, or |
| 3976 |
// used in logical operations. |
3976 |
// used in logical operations. |
| 3977 |
for (auto *CompareUse : Compare.getNode()->uses()) |
3977 |
for (auto *CompareUse : Compare.getNode()->uses()) |
| 3978 |
if (CompareUse->getOpcode() != ISD::SIGN_EXTEND && |
3978 |
if (CompareUse->getOpcode() != ISD::SIGN_EXTEND && |
| 3979 |
CompareUse->getOpcode() != ISD::ZERO_EXTEND && |
3979 |
CompareUse->getOpcode() != ISD::ZERO_EXTEND && |
| 3980 |
CompareUse->getOpcode() != ISD::SELECT && |
3980 |
CompareUse->getOpcode() != ISD::SELECT && |
| 3981 |
!ISD::isBitwiseLogicOp(CompareUse->getOpcode())) { |
3981 |
!ISD::isBitwiseLogicOp(CompareUse->getOpcode())) { |
| 3982 |
OmittedForNonExtendUses++; |
3982 |
OmittedForNonExtendUses++; |
| 3983 |
return false; |
3983 |
return false; |
| 3984 |
} |
3984 |
} |
| 3985 |
return true; |
3985 |
return true; |
| 3986 |
} |
3986 |
} |
| 3987 |
|
3987 |
|
| 3988 |
/// Returns an equivalent of a SETCC node but with the result the same width as |
3988 |
/// Returns an equivalent of a SETCC node but with the result the same width as |
| 3989 |
/// the inputs. This can also be used for SELECT_CC if either the true or false |
3989 |
/// the inputs. This can also be used for SELECT_CC if either the true or false |
| 3990 |
/// values is a power of two while the other is zero. |
3990 |
/// values is a power of two while the other is zero. |
| 3991 |
SDValue IntegerCompareEliminator::getSETCCInGPR(SDValue Compare, |
3991 |
SDValue IntegerCompareEliminator::getSETCCInGPR(SDValue Compare, |
| 3992 |
SetccInGPROpts ConvOpts) { |
3992 |
SetccInGPROpts ConvOpts) { |
| 3993 |
assert((Compare.getOpcode() == ISD::SETCC || |
3993 |
assert((Compare.getOpcode() == ISD::SETCC || |
| 3994 |
Compare.getOpcode() == ISD::SELECT_CC) && |
3994 |
Compare.getOpcode() == ISD::SELECT_CC) && |
| 3995 |
"An ISD::SETCC node required here."); |
3995 |
"An ISD::SETCC node required here."); |
| 3996 |
|
3996 |
|
| 3997 |
// Don't convert this comparison to a GPR sequence because there are uses |
3997 |
// Don't convert this comparison to a GPR sequence because there are uses |
| 3998 |
// of the i1 result (i.e. uses that require the result in the CR). |
3998 |
// of the i1 result (i.e. uses that require the result in the CR). |
| 3999 |
if ((Compare.getOpcode() == ISD::SETCC) && !allUsesExtend(Compare, CurDAG)) |
3999 |
if ((Compare.getOpcode() == ISD::SETCC) && !allUsesExtend(Compare, CurDAG)) |
| 4000 |
return SDValue(); |
4000 |
return SDValue(); |
| 4001 |
|
4001 |
|
| 4002 |
SDValue LHS = Compare.getOperand(0); |
4002 |
SDValue LHS = Compare.getOperand(0); |
| 4003 |
SDValue RHS = Compare.getOperand(1); |
4003 |
SDValue RHS = Compare.getOperand(1); |
| 4004 |
|
4004 |
|
| 4005 |
// The condition code is operand 2 for SETCC and operand 4 for SELECT_CC. |
4005 |
// The condition code is operand 2 for SETCC and operand 4 for SELECT_CC. |
| 4006 |
int CCOpNum = Compare.getOpcode() == ISD::SELECT_CC ? 4 : 2; |
4006 |
int CCOpNum = Compare.getOpcode() == ISD::SELECT_CC ? 4 : 2; |
| 4007 |
ISD::CondCode CC = |
4007 |
ISD::CondCode CC = |
| 4008 |
cast(Compare.getOperand(CCOpNum))->get(); |
4008 |
cast(Compare.getOperand(CCOpNum))->get(); |
| 4009 |
EVT InputVT = LHS.getValueType(); |
4009 |
EVT InputVT = LHS.getValueType(); |
| 4010 |
if (InputVT != MVT::i32 && InputVT != MVT::i64) |
4010 |
if (InputVT != MVT::i32 && InputVT != MVT::i64) |
| 4011 |
return SDValue(); |
4011 |
return SDValue(); |
| 4012 |
|
4012 |
|
| 4013 |
if (ConvOpts == SetccInGPROpts::ZExtInvert || |
4013 |
if (ConvOpts == SetccInGPROpts::ZExtInvert || |
| 4014 |
ConvOpts == SetccInGPROpts::SExtInvert) |
4014 |
ConvOpts == SetccInGPROpts::SExtInvert) |
| 4015 |
CC = ISD::getSetCCInverse(CC, InputVT); |
4015 |
CC = ISD::getSetCCInverse(CC, InputVT); |
| 4016 |
|
4016 |
|
| 4017 |
bool Inputs32Bit = InputVT == MVT::i32; |
4017 |
bool Inputs32Bit = InputVT == MVT::i32; |
| 4018 |
|
4018 |
|
| 4019 |
SDLoc dl(Compare); |
4019 |
SDLoc dl(Compare); |
| 4020 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
4020 |
ConstantSDNode *RHSConst = dyn_cast(RHS); |
| 4021 |
int64_t RHSValue = RHSConst ? RHSConst->getSExtValue() : INT64_MAX; |
4021 |
int64_t RHSValue = RHSConst ? RHSConst->getSExtValue() : INT64_MAX; |
| 4022 |
bool IsSext = ConvOpts == SetccInGPROpts::SExtOrig || |
4022 |
bool IsSext = ConvOpts == SetccInGPROpts::SExtOrig || |
| 4023 |
ConvOpts == SetccInGPROpts::SExtInvert; |
4023 |
ConvOpts == SetccInGPROpts::SExtInvert; |
| 4024 |
|
4024 |
|
| 4025 |
if (IsSext && Inputs32Bit) |
4025 |
if (IsSext && Inputs32Bit) |
| 4026 |
return get32BitSExtCompare(LHS, RHS, CC, RHSValue, dl); |
4026 |
return get32BitSExtCompare(LHS, RHS, CC, RHSValue, dl); |
| 4027 |
else if (Inputs32Bit) |
4027 |
else if (Inputs32Bit) |
| 4028 |
return get32BitZExtCompare(LHS, RHS, CC, RHSValue, dl); |
4028 |
return get32BitZExtCompare(LHS, RHS, CC, RHSValue, dl); |
| 4029 |
else if (IsSext) |
4029 |
else if (IsSext) |
| 4030 |
return get64BitSExtCompare(LHS, RHS, CC, RHSValue, dl); |
4030 |
return get64BitSExtCompare(LHS, RHS, CC, RHSValue, dl); |
| 4031 |
return get64BitZExtCompare(LHS, RHS, CC, RHSValue, dl); |
4031 |
return get64BitZExtCompare(LHS, RHS, CC, RHSValue, dl); |
| 4032 |
} |
4032 |
} |
| 4033 |
|
4033 |
|
| 4034 |
} // end anonymous namespace |
4034 |
} // end anonymous namespace |
| 4035 |
|
4035 |
|
| 4036 |
bool PPCDAGToDAGISel::tryIntCompareInGPR(SDNode *N) { |
4036 |
bool PPCDAGToDAGISel::tryIntCompareInGPR(SDNode *N) { |
| 4037 |
if (N->getValueType(0) != MVT::i32 && |
4037 |
if (N->getValueType(0) != MVT::i32 && |
| 4038 |
N->getValueType(0) != MVT::i64) |
4038 |
N->getValueType(0) != MVT::i64) |
| 4039 |
return false; |
4039 |
return false; |
| 4040 |
|
4040 |
|
| 4041 |
// This optimization will emit code that assumes 64-bit registers |
4041 |
// This optimization will emit code that assumes 64-bit registers |
| 4042 |
// so we don't want to run it in 32-bit mode. Also don't run it |
4042 |
// so we don't want to run it in 32-bit mode. Also don't run it |
| 4043 |
// on functions that are not to be optimized. |
4043 |
// on functions that are not to be optimized. |
| 4044 |
if (TM.getOptLevel() == CodeGenOpt::None || !TM.isPPC64()) |
4044 |
if (TM.getOptLevel() == CodeGenOpt::None || !TM.isPPC64()) |
| 4045 |
return false; |
4045 |
return false; |
| 4046 |
|
4046 |
|
| 4047 |
// For POWER10, it is more profitable to use the set boolean extension |
4047 |
// For POWER10, it is more profitable to use the set boolean extension |
| 4048 |
// instructions rather than the integer compare elimination codegen. |
4048 |
// instructions rather than the integer compare elimination codegen. |
| 4049 |
// Users can override this via the command line option, `--ppc-gpr-icmps`. |
4049 |
// Users can override this via the command line option, `--ppc-gpr-icmps`. |
| 4050 |
if (!(CmpInGPR.getNumOccurrences() > 0) && Subtarget->isISA3_1()) |
4050 |
if (!(CmpInGPR.getNumOccurrences() > 0) && Subtarget->isISA3_1()) |
| 4051 |
return false; |
4051 |
return false; |
| 4052 |
|
4052 |
|
| 4053 |
switch (N->getOpcode()) { |
4053 |
switch (N->getOpcode()) { |
| 4054 |
default: break; |
4054 |
default: break; |
| 4055 |
case ISD::ZERO_EXTEND: |
4055 |
case ISD::ZERO_EXTEND: |
| 4056 |
case ISD::SIGN_EXTEND: |
4056 |
case ISD::SIGN_EXTEND: |
| 4057 |
case ISD::AND: |
4057 |
case ISD::AND: |
| 4058 |
case ISD::OR: |
4058 |
case ISD::OR: |
| 4059 |
case ISD::XOR: { |
4059 |
case ISD::XOR: { |
| 4060 |
IntegerCompareEliminator ICmpElim(CurDAG, this); |
4060 |
IntegerCompareEliminator ICmpElim(CurDAG, this); |
| 4061 |
if (SDNode *New = ICmpElim.Select(N)) { |
4061 |
if (SDNode *New = ICmpElim.Select(N)) { |
| 4062 |
ReplaceNode(N, New); |
4062 |
ReplaceNode(N, New); |
| 4063 |
return true; |
4063 |
return true; |
| 4064 |
} |
4064 |
} |
| 4065 |
} |
4065 |
} |
| 4066 |
} |
4066 |
} |
| 4067 |
return false; |
4067 |
return false; |
| 4068 |
} |
4068 |
} |
| 4069 |
|
4069 |
|
| 4070 |
bool PPCDAGToDAGISel::tryBitPermutation(SDNode *N) { |
4070 |
bool PPCDAGToDAGISel::tryBitPermutation(SDNode *N) { |
| 4071 |
if (N->getValueType(0) != MVT::i32 && |
4071 |
if (N->getValueType(0) != MVT::i32 && |
| 4072 |
N->getValueType(0) != MVT::i64) |
4072 |
N->getValueType(0) != MVT::i64) |
| 4073 |
return false; |
4073 |
return false; |
| 4074 |
|
4074 |
|
| 4075 |
if (!UseBitPermRewriter) |
4075 |
if (!UseBitPermRewriter) |
| 4076 |
return false; |
4076 |
return false; |
| 4077 |
|
4077 |
|
| 4078 |
switch (N->getOpcode()) { |
4078 |
switch (N->getOpcode()) { |
| 4079 |
default: break; |
4079 |
default: break; |
| 4080 |
case ISD::SRL: |
4080 |
case ISD::SRL: |
| 4081 |
// If we are on P10, we have a pattern for 32-bit (srl (bswap r), 16) that |
4081 |
// If we are on P10, we have a pattern for 32-bit (srl (bswap r), 16) that |
| 4082 |
// uses the BRH instruction. |
4082 |
// uses the BRH instruction. |
| 4083 |
if (Subtarget->isISA3_1() && N->getValueType(0) == MVT::i32 && |
4083 |
if (Subtarget->isISA3_1() && N->getValueType(0) == MVT::i32 && |
| 4084 |
N->getOperand(0).getOpcode() == ISD::BSWAP) { |
4084 |
N->getOperand(0).getOpcode() == ISD::BSWAP) { |
| 4085 |
auto &OpRight = N->getOperand(1); |
4085 |
auto &OpRight = N->getOperand(1); |
| 4086 |
ConstantSDNode *SRLConst = dyn_cast(OpRight); |
4086 |
ConstantSDNode *SRLConst = dyn_cast(OpRight); |
| 4087 |
if (SRLConst && SRLConst->getSExtValue() == 16) |
4087 |
if (SRLConst && SRLConst->getSExtValue() == 16) |
| 4088 |
return false; |
4088 |
return false; |
| 4089 |
} |
4089 |
} |
| 4090 |
[[fallthrough]]; |
4090 |
[[fallthrough]]; |
| 4091 |
case ISD::ROTL: |
4091 |
case ISD::ROTL: |
| 4092 |
case ISD::SHL: |
4092 |
case ISD::SHL: |
| 4093 |
case ISD::AND: |
4093 |
case ISD::AND: |
| 4094 |
case ISD::OR: { |
4094 |
case ISD::OR: { |
| 4095 |
BitPermutationSelector BPS(CurDAG); |
4095 |
BitPermutationSelector BPS(CurDAG); |
| 4096 |
if (SDNode *New = BPS.Select(N)) { |
4096 |
if (SDNode *New = BPS.Select(N)) { |
| 4097 |
ReplaceNode(N, New); |
4097 |
ReplaceNode(N, New); |
| 4098 |
return true; |
4098 |
return true; |
| 4099 |
} |
4099 |
} |
| 4100 |
return false; |
4100 |
return false; |
| 4101 |
} |
4101 |
} |
| 4102 |
} |
4102 |
} |
| 4103 |
|
4103 |
|
| 4104 |
return false; |
4104 |
return false; |
| 4105 |
} |
4105 |
} |
| 4106 |
|
4106 |
|
| 4107 |
/// SelectCC - Select a comparison of the specified values with the specified |
4107 |
/// SelectCC - Select a comparison of the specified values with the specified |
| 4108 |
/// condition code, returning the CR# of the expression. |
4108 |
/// condition code, returning the CR# of the expression. |
| 4109 |
SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
4109 |
SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
| 4110 |
const SDLoc &dl, SDValue Chain) { |
4110 |
const SDLoc &dl, SDValue Chain) { |
| 4111 |
// Always select the LHS. |
4111 |
// Always select the LHS. |
| 4112 |
unsigned Opc; |
4112 |
unsigned Opc; |
| 4113 |
|
4113 |
|
| 4114 |
if (LHS.getValueType() == MVT::i32) { |
4114 |
if (LHS.getValueType() == MVT::i32) { |
| 4115 |
unsigned Imm; |
4115 |
unsigned Imm; |
| 4116 |
if (CC == ISD::SETEQ || CC == ISD::SETNE) { |
4116 |
if (CC == ISD::SETEQ || CC == ISD::SETNE) { |
| 4117 |
if (isInt32Immediate(RHS, Imm)) { |
4117 |
if (isInt32Immediate(RHS, Imm)) { |
| 4118 |
// SETEQ/SETNE comparison with 16-bit immediate, fold it. |
4118 |
// SETEQ/SETNE comparison with 16-bit immediate, fold it. |
| 4119 |
if (isUInt<16>(Imm)) |
4119 |
if (isUInt<16>(Imm)) |
| 4120 |
return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS, |
4120 |
return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS, |
| 4121 |
getI32Imm(Imm & 0xFFFF, dl)), |
4121 |
getI32Imm(Imm & 0xFFFF, dl)), |
| 4122 |
0); |
4122 |
0); |
| 4123 |
// If this is a 16-bit signed immediate, fold it. |
4123 |
// If this is a 16-bit signed immediate, fold it. |
| 4124 |
if (isInt<16>((int)Imm)) |
4124 |
if (isInt<16>((int)Imm)) |
| 4125 |
return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, |
4125 |
return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, |
| 4126 |
getI32Imm(Imm & 0xFFFF, dl)), |
4126 |
getI32Imm(Imm & 0xFFFF, dl)), |
| 4127 |
0); |
4127 |
0); |
| 4128 |
|
4128 |
|
| 4129 |
// For non-equality comparisons, the default code would materialize the |
4129 |
// For non-equality comparisons, the default code would materialize the |
| 4130 |
// constant, then compare against it, like this: |
4130 |
// constant, then compare against it, like this: |
| 4131 |
// lis r2, 4660 |
4131 |
// lis r2, 4660 |
| 4132 |
// ori r2, r2, 22136 |
4132 |
// ori r2, r2, 22136 |
| 4133 |
// cmpw cr0, r3, r2 |
4133 |
// cmpw cr0, r3, r2 |
| 4134 |
// Since we are just comparing for equality, we can emit this instead: |
4134 |
// Since we are just comparing for equality, we can emit this instead: |
| 4135 |
// xoris r0,r3,0x1234 |
4135 |
// xoris r0,r3,0x1234 |
| 4136 |
// cmplwi cr0,r0,0x5678 |
4136 |
// cmplwi cr0,r0,0x5678 |
| 4137 |
// beq cr0,L6 |
4137 |
// beq cr0,L6 |
| 4138 |
SDValue Xor(CurDAG->getMachineNode(PPC::XORIS, dl, MVT::i32, LHS, |
4138 |
SDValue Xor(CurDAG->getMachineNode(PPC::XORIS, dl, MVT::i32, LHS, |
| 4139 |
getI32Imm(Imm >> 16, dl)), 0); |
4139 |
getI32Imm(Imm >> 16, dl)), 0); |
| 4140 |
return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, Xor, |
4140 |
return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, Xor, |
| 4141 |
getI32Imm(Imm & 0xFFFF, dl)), 0); |
4141 |
getI32Imm(Imm & 0xFFFF, dl)), 0); |
| 4142 |
} |
4142 |
} |
| 4143 |
Opc = PPC::CMPLW; |
4143 |
Opc = PPC::CMPLW; |
| 4144 |
} else if (ISD::isUnsignedIntSetCC(CC)) { |
4144 |
} else if (ISD::isUnsignedIntSetCC(CC)) { |
| 4145 |
if (isInt32Immediate(RHS, Imm) && isUInt<16>(Imm)) |
4145 |
if (isInt32Immediate(RHS, Imm) && isUInt<16>(Imm)) |
| 4146 |
return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS, |
4146 |
return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS, |
| 4147 |
getI32Imm(Imm & 0xFFFF, dl)), 0); |
4147 |
getI32Imm(Imm & 0xFFFF, dl)), 0); |
| 4148 |
Opc = PPC::CMPLW; |
4148 |
Opc = PPC::CMPLW; |
| 4149 |
} else { |
4149 |
} else { |
| 4150 |
int16_t SImm; |
4150 |
int16_t SImm; |
| 4151 |
if (isIntS16Immediate(RHS, SImm)) |
4151 |
if (isIntS16Immediate(RHS, SImm)) |
| 4152 |
return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, |
4152 |
return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, |
| 4153 |
getI32Imm((int)SImm & 0xFFFF, |
4153 |
getI32Imm((int)SImm & 0xFFFF, |
| 4154 |
dl)), |
4154 |
dl)), |
| 4155 |
0); |
4155 |
0); |
| 4156 |
Opc = PPC::CMPW; |
4156 |
Opc = PPC::CMPW; |
| 4157 |
} |
4157 |
} |
| 4158 |
} else if (LHS.getValueType() == MVT::i64) { |
4158 |
} else if (LHS.getValueType() == MVT::i64) { |
| 4159 |
uint64_t Imm; |
4159 |
uint64_t Imm; |
| 4160 |
if (CC == ISD::SETEQ || CC == ISD::SETNE) { |
4160 |
if (CC == ISD::SETEQ || CC == ISD::SETNE) { |
| 4161 |
if (isInt64Immediate(RHS.getNode(), Imm)) { |
4161 |
if (isInt64Immediate(RHS.getNode(), Imm)) { |
| 4162 |
// SETEQ/SETNE comparison with 16-bit immediate, fold it. |
4162 |
// SETEQ/SETNE comparison with 16-bit immediate, fold it. |
| 4163 |
if (isUInt<16>(Imm)) |
4163 |
if (isUInt<16>(Imm)) |
| 4164 |
return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS, |
4164 |
return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS, |
| 4165 |
getI32Imm(Imm & 0xFFFF, dl)), |
4165 |
getI32Imm(Imm & 0xFFFF, dl)), |
| 4166 |
0); |
4166 |
0); |
| 4167 |
// If this is a 16-bit signed immediate, fold it. |
4167 |
// If this is a 16-bit signed immediate, fold it. |
| 4168 |
if (isInt<16>(Imm)) |
4168 |
if (isInt<16>(Imm)) |
| 4169 |
return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, |
4169 |
return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, |
| 4170 |
getI32Imm(Imm & 0xFFFF, dl)), |
4170 |
getI32Imm(Imm & 0xFFFF, dl)), |
| 4171 |
0); |
4171 |
0); |
| 4172 |
|
4172 |
|
| 4173 |
// For non-equality comparisons, the default code would materialize the |
4173 |
// For non-equality comparisons, the default code would materialize the |
| 4174 |
// constant, then compare against it, like this: |
4174 |
// constant, then compare against it, like this: |
| 4175 |
// lis r2, 4660 |
4175 |
// lis r2, 4660 |
| 4176 |
// ori r2, r2, 22136 |
4176 |
// ori r2, r2, 22136 |
| 4177 |
// cmpd cr0, r3, r2 |
4177 |
// cmpd cr0, r3, r2 |
| 4178 |
// Since we are just comparing for equality, we can emit this instead: |
4178 |
// Since we are just comparing for equality, we can emit this instead: |
| 4179 |
// xoris r0,r3,0x1234 |
4179 |
// xoris r0,r3,0x1234 |
| 4180 |
// cmpldi cr0,r0,0x5678 |
4180 |
// cmpldi cr0,r0,0x5678 |
| 4181 |
// beq cr0,L6 |
4181 |
// beq cr0,L6 |
| 4182 |
if (isUInt<32>(Imm)) { |
4182 |
if (isUInt<32>(Imm)) { |
| 4183 |
SDValue Xor(CurDAG->getMachineNode(PPC::XORIS8, dl, MVT::i64, LHS, |
4183 |
SDValue Xor(CurDAG->getMachineNode(PPC::XORIS8, dl, MVT::i64, LHS, |
| 4184 |
getI64Imm(Imm >> 16, dl)), 0); |
4184 |
getI64Imm(Imm >> 16, dl)), 0); |
| 4185 |
return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, Xor, |
4185 |
return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, Xor, |
| 4186 |
getI64Imm(Imm & 0xFFFF, dl)), |
4186 |
getI64Imm(Imm & 0xFFFF, dl)), |
| 4187 |
0); |
4187 |
0); |
| 4188 |
} |
4188 |
} |
| 4189 |
} |
4189 |
} |
| 4190 |
Opc = PPC::CMPLD; |
4190 |
Opc = PPC::CMPLD; |
| 4191 |
} else if (ISD::isUnsignedIntSetCC(CC)) { |
4191 |
} else if (ISD::isUnsignedIntSetCC(CC)) { |
| 4192 |
if (isInt64Immediate(RHS.getNode(), Imm) && isUInt<16>(Imm)) |
4192 |
if (isInt64Immediate(RHS.getNode(), Imm) && isUInt<16>(Imm)) |
| 4193 |
return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS, |
4193 |
return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS, |
| 4194 |
getI64Imm(Imm & 0xFFFF, dl)), 0); |
4194 |
getI64Imm(Imm & 0xFFFF, dl)), 0); |
| 4195 |
Opc = PPC::CMPLD; |
4195 |
Opc = PPC::CMPLD; |
| 4196 |
} else { |
4196 |
} else { |
| 4197 |
int16_t SImm; |
4197 |
int16_t SImm; |
| 4198 |
if (isIntS16Immediate(RHS, SImm)) |
4198 |
if (isIntS16Immediate(RHS, SImm)) |
| 4199 |
return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, |
4199 |
return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, |
| 4200 |
getI64Imm(SImm & 0xFFFF, dl)), |
4200 |
getI64Imm(SImm & 0xFFFF, dl)), |
| 4201 |
0); |
4201 |
0); |
| 4202 |
Opc = PPC::CMPD; |
4202 |
Opc = PPC::CMPD; |
| 4203 |
} |
4203 |
} |
| 4204 |
} else if (LHS.getValueType() == MVT::f32) { |
4204 |
} else if (LHS.getValueType() == MVT::f32) { |
| 4205 |
if (Subtarget->hasSPE()) { |
4205 |
if (Subtarget->hasSPE()) { |
| 4206 |
switch (CC) { |
4206 |
switch (CC) { |
| 4207 |
default: |
4207 |
default: |
| 4208 |
case ISD::SETEQ: |
4208 |
case ISD::SETEQ: |
| 4209 |
case ISD::SETNE: |
4209 |
case ISD::SETNE: |
| 4210 |
Opc = PPC::EFSCMPEQ; |
4210 |
Opc = PPC::EFSCMPEQ; |
| 4211 |
break; |
4211 |
break; |
| 4212 |
case ISD::SETLT: |
4212 |
case ISD::SETLT: |
| 4213 |
case ISD::SETGE: |
4213 |
case ISD::SETGE: |
| 4214 |
case ISD::SETOLT: |
4214 |
case ISD::SETOLT: |
| 4215 |
case ISD::SETOGE: |
4215 |
case ISD::SETOGE: |
| 4216 |
case ISD::SETULT: |
4216 |
case ISD::SETULT: |
| 4217 |
case ISD::SETUGE: |
4217 |
case ISD::SETUGE: |
| 4218 |
Opc = PPC::EFSCMPLT; |
4218 |
Opc = PPC::EFSCMPLT; |
| 4219 |
break; |
4219 |
break; |
| 4220 |
case ISD::SETGT: |
4220 |
case ISD::SETGT: |
| 4221 |
case ISD::SETLE: |
4221 |
case ISD::SETLE: |
| 4222 |
case ISD::SETOGT: |
4222 |
case ISD::SETOGT: |
| 4223 |
case ISD::SETOLE: |
4223 |
case ISD::SETOLE: |
| 4224 |
case ISD::SETUGT: |
4224 |
case ISD::SETUGT: |
| 4225 |
case ISD::SETULE: |
4225 |
case ISD::SETULE: |
| 4226 |
Opc = PPC::EFSCMPGT; |
4226 |
Opc = PPC::EFSCMPGT; |
| 4227 |
break; |
4227 |
break; |
| 4228 |
} |
4228 |
} |
| 4229 |
} else |
4229 |
} else |
| 4230 |
Opc = PPC::FCMPUS; |
4230 |
Opc = PPC::FCMPUS; |
| 4231 |
} else if (LHS.getValueType() == MVT::f64) { |
4231 |
} else if (LHS.getValueType() == MVT::f64) { |
| 4232 |
if (Subtarget->hasSPE()) { |
4232 |
if (Subtarget->hasSPE()) { |
| 4233 |
switch (CC) { |
4233 |
switch (CC) { |
| 4234 |
default: |
4234 |
default: |
| 4235 |
case ISD::SETEQ: |
4235 |
case ISD::SETEQ: |
| 4236 |
case ISD::SETNE: |
4236 |
case ISD::SETNE: |
| 4237 |
Opc = PPC::EFDCMPEQ; |
4237 |
Opc = PPC::EFDCMPEQ; |
| 4238 |
break; |
4238 |
break; |
| 4239 |
case ISD::SETLT: |
4239 |
case ISD::SETLT: |
| 4240 |
case ISD::SETGE: |
4240 |
case ISD::SETGE: |
| 4241 |
case ISD::SETOLT: |
4241 |
case ISD::SETOLT: |
| 4242 |
case ISD::SETOGE: |
4242 |
case ISD::SETOGE: |
| 4243 |
case ISD::SETULT: |
4243 |
case ISD::SETULT: |
| 4244 |
case ISD::SETUGE: |
4244 |
case ISD::SETUGE: |
| 4245 |
Opc = PPC::EFDCMPLT; |
4245 |
Opc = PPC::EFDCMPLT; |
| 4246 |
break; |
4246 |
break; |
| 4247 |
case ISD::SETGT: |
4247 |
case ISD::SETGT: |
| 4248 |
case ISD::SETLE: |
4248 |
case ISD::SETLE: |
| 4249 |
case ISD::SETOGT: |
4249 |
case ISD::SETOGT: |
| 4250 |
case ISD::SETOLE: |
4250 |
case ISD::SETOLE: |
| 4251 |
case ISD::SETUGT: |
4251 |
case ISD::SETUGT: |
| 4252 |
case ISD::SETULE: |
4252 |
case ISD::SETULE: |
| 4253 |
Opc = PPC::EFDCMPGT; |
4253 |
Opc = PPC::EFDCMPGT; |
| 4254 |
break; |
4254 |
break; |
| 4255 |
} |
4255 |
} |
| 4256 |
} else |
4256 |
} else |
| 4257 |
Opc = Subtarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD; |
4257 |
Opc = Subtarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD; |
| 4258 |
} else { |
4258 |
} else { |
| 4259 |
assert(LHS.getValueType() == MVT::f128 && "Unknown vt!"); |
4259 |
assert(LHS.getValueType() == MVT::f128 && "Unknown vt!"); |
| 4260 |
assert(Subtarget->hasP9Vector() && "XSCMPUQP requires Power9 Vector"); |
4260 |
assert(Subtarget->hasP9Vector() && "XSCMPUQP requires Power9 Vector"); |
| 4261 |
Opc = PPC::XSCMPUQP; |
4261 |
Opc = PPC::XSCMPUQP; |
| 4262 |
} |
4262 |
} |
| 4263 |
if (Chain) |
4263 |
if (Chain) |
| 4264 |
return SDValue( |
4264 |
return SDValue( |
| 4265 |
CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::Other, LHS, RHS, Chain), |
4265 |
CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::Other, LHS, RHS, Chain), |
| 4266 |
0); |
4266 |
0); |
| 4267 |
else |
4267 |
else |
| 4268 |
return SDValue(CurDAG->getMachineNode(Opc, dl, MVT::i32, LHS, RHS), 0); |
4268 |
return SDValue(CurDAG->getMachineNode(Opc, dl, MVT::i32, LHS, RHS), 0); |
| 4269 |
} |
4269 |
} |
| 4270 |
|
4270 |
|
| 4271 |
static PPC::Predicate getPredicateForSetCC(ISD::CondCode CC, const EVT &VT, |
4271 |
static PPC::Predicate getPredicateForSetCC(ISD::CondCode CC, const EVT &VT, |
| 4272 |
const PPCSubtarget *Subtarget) { |
4272 |
const PPCSubtarget *Subtarget) { |
| 4273 |
// For SPE instructions, the result is in GT bit of the CR |
4273 |
// For SPE instructions, the result is in GT bit of the CR |
| 4274 |
bool UseSPE = Subtarget->hasSPE() && VT.isFloatingPoint(); |
4274 |
bool UseSPE = Subtarget->hasSPE() && VT.isFloatingPoint(); |
| 4275 |
|
4275 |
|
| 4276 |
switch (CC) { |
4276 |
switch (CC) { |
| 4277 |
case ISD::SETUEQ: |
4277 |
case ISD::SETUEQ: |
| 4278 |
case ISD::SETONE: |
4278 |
case ISD::SETONE: |
| 4279 |
case ISD::SETOLE: |
4279 |
case ISD::SETOLE: |
| 4280 |
case ISD::SETOGE: |
4280 |
case ISD::SETOGE: |
| 4281 |
llvm_unreachable("Should be lowered by legalize!"); |
4281 |
llvm_unreachable("Should be lowered by legalize!"); |
| 4282 |
default: llvm_unreachable("Unknown condition!"); |
4282 |
default: llvm_unreachable("Unknown condition!"); |
| 4283 |
case ISD::SETOEQ: |
4283 |
case ISD::SETOEQ: |
| 4284 |
case ISD::SETEQ: |
4284 |
case ISD::SETEQ: |
| 4285 |
return UseSPE ? PPC::PRED_GT : PPC::PRED_EQ; |
4285 |
return UseSPE ? PPC::PRED_GT : PPC::PRED_EQ; |
| 4286 |
case ISD::SETUNE: |
4286 |
case ISD::SETUNE: |
| 4287 |
case ISD::SETNE: |
4287 |
case ISD::SETNE: |
| 4288 |
return UseSPE ? PPC::PRED_LE : PPC::PRED_NE; |
4288 |
return UseSPE ? PPC::PRED_LE : PPC::PRED_NE; |
| 4289 |
case ISD::SETOLT: |
4289 |
case ISD::SETOLT: |
| 4290 |
case ISD::SETLT: |
4290 |
case ISD::SETLT: |
| 4291 |
return UseSPE ? PPC::PRED_GT : PPC::PRED_LT; |
4291 |
return UseSPE ? PPC::PRED_GT : PPC::PRED_LT; |
| 4292 |
case ISD::SETULE: |
4292 |
case ISD::SETULE: |
| 4293 |
case ISD::SETLE: |
4293 |
case ISD::SETLE: |
| 4294 |
return PPC::PRED_LE; |
4294 |
return PPC::PRED_LE; |
| 4295 |
case ISD::SETOGT: |
4295 |
case ISD::SETOGT: |
| 4296 |
case ISD::SETGT: |
4296 |
case ISD::SETGT: |
| 4297 |
return PPC::PRED_GT; |
4297 |
return PPC::PRED_GT; |
| 4298 |
case ISD::SETUGE: |
4298 |
case ISD::SETUGE: |
| 4299 |
case ISD::SETGE: |
4299 |
case ISD::SETGE: |
| 4300 |
return UseSPE ? PPC::PRED_LE : PPC::PRED_GE; |
4300 |
return UseSPE ? PPC::PRED_LE : PPC::PRED_GE; |
| 4301 |
case ISD::SETO: return PPC::PRED_NU; |
4301 |
case ISD::SETO: return PPC::PRED_NU; |
| 4302 |
case ISD::SETUO: return PPC::PRED_UN; |
4302 |
case ISD::SETUO: return PPC::PRED_UN; |
| 4303 |
// These two are invalid for floating point. Assume we have int. |
4303 |
// These two are invalid for floating point. Assume we have int. |
| 4304 |
case ISD::SETULT: return PPC::PRED_LT; |
4304 |
case ISD::SETULT: return PPC::PRED_LT; |
| 4305 |
case ISD::SETUGT: return PPC::PRED_GT; |
4305 |
case ISD::SETUGT: return PPC::PRED_GT; |
| 4306 |
} |
4306 |
} |
| 4307 |
} |
4307 |
} |
| 4308 |
|
4308 |
|
| 4309 |
/// getCRIdxForSetCC - Return the index of the condition register field |
4309 |
/// getCRIdxForSetCC - Return the index of the condition register field |
| 4310 |
/// associated with the SetCC condition, and whether or not the field is |
4310 |
/// associated with the SetCC condition, and whether or not the field is |
| 4311 |
/// treated as inverted. That is, lt = 0; ge = 0 inverted. |
4311 |
/// treated as inverted. That is, lt = 0; ge = 0 inverted. |
| 4312 |
static unsigned getCRIdxForSetCC(ISD::CondCode CC, bool &Invert) { |
4312 |
static unsigned getCRIdxForSetCC(ISD::CondCode CC, bool &Invert) { |
| 4313 |
Invert = false; |
4313 |
Invert = false; |
| 4314 |
switch (CC) { |
4314 |
switch (CC) { |
| 4315 |
default: llvm_unreachable("Unknown condition!"); |
4315 |
default: llvm_unreachable("Unknown condition!"); |
| 4316 |
case ISD::SETOLT: |
4316 |
case ISD::SETOLT: |
| 4317 |
case ISD::SETLT: return 0; // Bit #0 = SETOLT |
4317 |
case ISD::SETLT: return 0; // Bit #0 = SETOLT |
| 4318 |
case ISD::SETOGT: |
4318 |
case ISD::SETOGT: |
| 4319 |
case ISD::SETGT: return 1; // Bit #1 = SETOGT |
4319 |
case ISD::SETGT: return 1; // Bit #1 = SETOGT |
| 4320 |
case ISD::SETOEQ: |
4320 |
case ISD::SETOEQ: |
| 4321 |
case ISD::SETEQ: return 2; // Bit #2 = SETOEQ |
4321 |
case ISD::SETEQ: return 2; // Bit #2 = SETOEQ |
| 4322 |
case ISD::SETUO: return 3; // Bit #3 = SETUO |
4322 |
case ISD::SETUO: return 3; // Bit #3 = SETUO |
| 4323 |
case ISD::SETUGE: |
4323 |
case ISD::SETUGE: |
| 4324 |
case ISD::SETGE: Invert = true; return 0; // !Bit #0 = SETUGE |
4324 |
case ISD::SETGE: Invert = true; return 0; // !Bit #0 = SETUGE |
| 4325 |
case ISD::SETULE: |
4325 |
case ISD::SETULE: |
| 4326 |
case ISD::SETLE: Invert = true; return 1; // !Bit #1 = SETULE |
4326 |
case ISD::SETLE: Invert = true; return 1; // !Bit #1 = SETULE |
| 4327 |
case ISD::SETUNE: |
4327 |
case ISD::SETUNE: |
| 4328 |
case ISD::SETNE: Invert = true; return 2; // !Bit #2 = SETUNE |
4328 |
case ISD::SETNE: Invert = true; return 2; // !Bit #2 = SETUNE |
| 4329 |
case ISD::SETO: Invert = true; return 3; // !Bit #3 = SETO |
4329 |
case ISD::SETO: Invert = true; return 3; // !Bit #3 = SETO |
| 4330 |
case ISD::SETUEQ: |
4330 |
case ISD::SETUEQ: |
| 4331 |
case ISD::SETOGE: |
4331 |
case ISD::SETOGE: |
| 4332 |
case ISD::SETOLE: |
4332 |
case ISD::SETOLE: |
| 4333 |
case ISD::SETONE: |
4333 |
case ISD::SETONE: |
| 4334 |
llvm_unreachable("Invalid branch code: should be expanded by legalize"); |
4334 |
llvm_unreachable("Invalid branch code: should be expanded by legalize"); |
| 4335 |
// These are invalid for floating point. Assume integer. |
4335 |
// These are invalid for floating point. Assume integer. |
| 4336 |
case ISD::SETULT: return 0; |
4336 |
case ISD::SETULT: return 0; |
| 4337 |
case ISD::SETUGT: return 1; |
4337 |
case ISD::SETUGT: return 1; |
| 4338 |
} |
4338 |
} |
| 4339 |
} |
4339 |
} |
| 4340 |
|
4340 |
|
| 4341 |
// getVCmpInst: return the vector compare instruction for the specified |
4341 |
// getVCmpInst: return the vector compare instruction for the specified |
| 4342 |
// vector type and condition code. Since this is for altivec specific code, |
4342 |
// vector type and condition code. Since this is for altivec specific code, |
| 4343 |
// only support the altivec types (v16i8, v8i16, v4i32, v2i64, v1i128, |
4343 |
// only support the altivec types (v16i8, v8i16, v4i32, v2i64, v1i128, |
| 4344 |
// and v4f32). |
4344 |
// and v4f32). |
| 4345 |
static unsigned int getVCmpInst(MVT VecVT, ISD::CondCode CC, |
4345 |
static unsigned int getVCmpInst(MVT VecVT, ISD::CondCode CC, |
| 4346 |
bool HasVSX, bool &Swap, bool &Negate) { |
4346 |
bool HasVSX, bool &Swap, bool &Negate) { |
| 4347 |
Swap = false; |
4347 |
Swap = false; |
| 4348 |
Negate = false; |
4348 |
Negate = false; |
| 4349 |
|
4349 |
|
| 4350 |
if (VecVT.isFloatingPoint()) { |
4350 |
if (VecVT.isFloatingPoint()) { |
| 4351 |
/* Handle some cases by swapping input operands. */ |
4351 |
/* Handle some cases by swapping input operands. */ |
| 4352 |
switch (CC) { |
4352 |
switch (CC) { |
| 4353 |
case ISD::SETLE: CC = ISD::SETGE; Swap = true; break; |
4353 |
case ISD::SETLE: CC = ISD::SETGE; Swap = true; break; |
| 4354 |
case ISD::SETLT: CC = ISD::SETGT; Swap = true; break; |
4354 |
case ISD::SETLT: CC = ISD::SETGT; Swap = true; break; |
| 4355 |
case ISD::SETOLE: CC = ISD::SETOGE; Swap = true; break; |
4355 |
case ISD::SETOLE: CC = ISD::SETOGE; Swap = true; break; |
| 4356 |
case ISD::SETOLT: CC = ISD::SETOGT; Swap = true; break; |
4356 |
case ISD::SETOLT: CC = ISD::SETOGT; Swap = true; break; |
| 4357 |
case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break; |
4357 |
case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break; |
| 4358 |
case ISD::SETUGT: CC = ISD::SETULT; Swap = true; break; |
4358 |
case ISD::SETUGT: CC = ISD::SETULT; Swap = true; break; |
| 4359 |
default: break; |
4359 |
default: break; |
| 4360 |
} |
4360 |
} |
| 4361 |
/* Handle some cases by negating the result. */ |
4361 |
/* Handle some cases by negating the result. */ |
| 4362 |
switch (CC) { |
4362 |
switch (CC) { |
| 4363 |
case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break; |
4363 |
case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break; |
| 4364 |
case ISD::SETUNE: CC = ISD::SETOEQ; Negate = true; break; |
4364 |
case ISD::SETUNE: CC = ISD::SETOEQ; Negate = true; break; |
| 4365 |
case ISD::SETULE: CC = ISD::SETOGT; Negate = true; break; |
4365 |
case ISD::SETULE: CC = ISD::SETOGT; Negate = true; break; |
| 4366 |
case ISD::SETULT: CC = ISD::SETOGE; Negate = true; break; |
4366 |
case ISD::SETULT: CC = ISD::SETOGE; Negate = true; break; |
| 4367 |
default: break; |
4367 |
default: break; |
| 4368 |
} |
4368 |
} |
| 4369 |
/* We have instructions implementing the remaining cases. */ |
4369 |
/* We have instructions implementing the remaining cases. */ |
| 4370 |
switch (CC) { |
4370 |
switch (CC) { |
| 4371 |
case ISD::SETEQ: |
4371 |
case ISD::SETEQ: |
| 4372 |
case ISD::SETOEQ: |
4372 |
case ISD::SETOEQ: |
| 4373 |
if (VecVT == MVT::v4f32) |
4373 |
if (VecVT == MVT::v4f32) |
| 4374 |
return HasVSX ? PPC::XVCMPEQSP : PPC::VCMPEQFP; |
4374 |
return HasVSX ? PPC::XVCMPEQSP : PPC::VCMPEQFP; |
| 4375 |
else if (VecVT == MVT::v2f64) |
4375 |
else if (VecVT == MVT::v2f64) |
| 4376 |
return PPC::XVCMPEQDP; |
4376 |
return PPC::XVCMPEQDP; |
| 4377 |
break; |
4377 |
break; |
| 4378 |
case ISD::SETGT: |
4378 |
case ISD::SETGT: |
| 4379 |
case ISD::SETOGT: |
4379 |
case ISD::SETOGT: |
| 4380 |
if (VecVT == MVT::v4f32) |
4380 |
if (VecVT == MVT::v4f32) |
| 4381 |
return HasVSX ? PPC::XVCMPGTSP : PPC::VCMPGTFP; |
4381 |
return HasVSX ? PPC::XVCMPGTSP : PPC::VCMPGTFP; |
| 4382 |
else if (VecVT == MVT::v2f64) |
4382 |
else if (VecVT == MVT::v2f64) |
| 4383 |
return PPC::XVCMPGTDP; |
4383 |
return PPC::XVCMPGTDP; |
| 4384 |
break; |
4384 |
break; |
| 4385 |
case ISD::SETGE: |
4385 |
case ISD::SETGE: |
| 4386 |
case ISD::SETOGE: |
4386 |
case ISD::SETOGE: |
| 4387 |
if (VecVT == MVT::v4f32) |
4387 |
if (VecVT == MVT::v4f32) |
| 4388 |
return HasVSX ? PPC::XVCMPGESP : PPC::VCMPGEFP; |
4388 |
return HasVSX ? PPC::XVCMPGESP : PPC::VCMPGEFP; |
| 4389 |
else if (VecVT == MVT::v2f64) |
4389 |
else if (VecVT == MVT::v2f64) |
| 4390 |
return PPC::XVCMPGEDP; |
4390 |
return PPC::XVCMPGEDP; |
| 4391 |
break; |
4391 |
break; |
| 4392 |
default: |
4392 |
default: |
| 4393 |
break; |
4393 |
break; |
| 4394 |
} |
4394 |
} |
| 4395 |
llvm_unreachable("Invalid floating-point vector compare condition"); |
4395 |
llvm_unreachable("Invalid floating-point vector compare condition"); |
| 4396 |
} else { |
4396 |
} else { |
| 4397 |
/* Handle some cases by swapping input operands. */ |
4397 |
/* Handle some cases by swapping input operands. */ |
| 4398 |
switch (CC) { |
4398 |
switch (CC) { |
| 4399 |
case ISD::SETGE: CC = ISD::SETLE; Swap = true; break; |
4399 |
case ISD::SETGE: CC = ISD::SETLE; Swap = true; break; |
| 4400 |
case ISD::SETLT: CC = ISD::SETGT; Swap = true; break; |
4400 |
case ISD::SETLT: CC = ISD::SETGT; Swap = true; break; |
| 4401 |
case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break; |
4401 |
case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break; |
| 4402 |
case ISD::SETULT: CC = ISD::SETUGT; Swap = true; break; |
4402 |
case ISD::SETULT: CC = ISD::SETUGT; Swap = true; break; |
| 4403 |
default: break; |
4403 |
default: break; |
| 4404 |
} |
4404 |
} |
| 4405 |
/* Handle some cases by negating the result. */ |
4405 |
/* Handle some cases by negating the result. */ |
| 4406 |
switch (CC) { |
4406 |
switch (CC) { |
| 4407 |
case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break; |
4407 |
case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break; |
| 4408 |
case ISD::SETUNE: CC = ISD::SETUEQ; Negate = true; break; |
4408 |
case ISD::SETUNE: CC = ISD::SETUEQ; Negate = true; break; |
| 4409 |
case ISD::SETLE: CC = ISD::SETGT; Negate = true; break; |
4409 |
case ISD::SETLE: CC = ISD::SETGT; Negate = true; break; |
| 4410 |
case ISD::SETULE: CC = ISD::SETUGT; Negate = true; break; |
4410 |
case ISD::SETULE: CC = ISD::SETUGT; Negate = true; break; |
| 4411 |
default: break; |
4411 |
default: break; |
| 4412 |
} |
4412 |
} |
| 4413 |
/* We have instructions implementing the remaining cases. */ |
4413 |
/* We have instructions implementing the remaining cases. */ |
| 4414 |
switch (CC) { |
4414 |
switch (CC) { |
| 4415 |
case ISD::SETEQ: |
4415 |
case ISD::SETEQ: |
| 4416 |
case ISD::SETUEQ: |
4416 |
case ISD::SETUEQ: |
| 4417 |
if (VecVT == MVT::v16i8) |
4417 |
if (VecVT == MVT::v16i8) |
| 4418 |
return PPC::VCMPEQUB; |
4418 |
return PPC::VCMPEQUB; |
| 4419 |
else if (VecVT == MVT::v8i16) |
4419 |
else if (VecVT == MVT::v8i16) |
| 4420 |
return PPC::VCMPEQUH; |
4420 |
return PPC::VCMPEQUH; |
| 4421 |
else if (VecVT == MVT::v4i32) |
4421 |
else if (VecVT == MVT::v4i32) |
| 4422 |
return PPC::VCMPEQUW; |
4422 |
return PPC::VCMPEQUW; |
| 4423 |
else if (VecVT == MVT::v2i64) |
4423 |
else if (VecVT == MVT::v2i64) |
| 4424 |
return PPC::VCMPEQUD; |
4424 |
return PPC::VCMPEQUD; |
| 4425 |
else if (VecVT == MVT::v1i128) |
4425 |
else if (VecVT == MVT::v1i128) |
| 4426 |
return PPC::VCMPEQUQ; |
4426 |
return PPC::VCMPEQUQ; |
| 4427 |
break; |
4427 |
break; |
| 4428 |
case ISD::SETGT: |
4428 |
case ISD::SETGT: |
| 4429 |
if (VecVT == MVT::v16i8) |
4429 |
if (VecVT == MVT::v16i8) |
| 4430 |
return PPC::VCMPGTSB; |
4430 |
return PPC::VCMPGTSB; |
| 4431 |
else if (VecVT == MVT::v8i16) |
4431 |
else if (VecVT == MVT::v8i16) |
| 4432 |
return PPC::VCMPGTSH; |
4432 |
return PPC::VCMPGTSH; |
| 4433 |
else if (VecVT == MVT::v4i32) |
4433 |
else if (VecVT == MVT::v4i32) |
| 4434 |
return PPC::VCMPGTSW; |
4434 |
return PPC::VCMPGTSW; |
| 4435 |
else if (VecVT == MVT::v2i64) |
4435 |
else if (VecVT == MVT::v2i64) |
| 4436 |
return PPC::VCMPGTSD; |
4436 |
return PPC::VCMPGTSD; |
| 4437 |
else if (VecVT == MVT::v1i128) |
4437 |
else if (VecVT == MVT::v1i128) |
| 4438 |
return PPC::VCMPGTSQ; |
4438 |
return PPC::VCMPGTSQ; |
| 4439 |
break; |
4439 |
break; |
| 4440 |
case ISD::SETUGT: |
4440 |
case ISD::SETUGT: |
| 4441 |
if (VecVT == MVT::v16i8) |
4441 |
if (VecVT == MVT::v16i8) |
| 4442 |
return PPC::VCMPGTUB; |
4442 |
return PPC::VCMPGTUB; |
| 4443 |
else if (VecVT == MVT::v8i16) |
4443 |
else if (VecVT == MVT::v8i16) |
| 4444 |
return PPC::VCMPGTUH; |
4444 |
return PPC::VCMPGTUH; |
| 4445 |
else if (VecVT == MVT::v4i32) |
4445 |
else if (VecVT == MVT::v4i32) |
| 4446 |
return PPC::VCMPGTUW; |
4446 |
return PPC::VCMPGTUW; |
| 4447 |
else if (VecVT == MVT::v2i64) |
4447 |
else if (VecVT == MVT::v2i64) |
| 4448 |
return PPC::VCMPGTUD; |
4448 |
return PPC::VCMPGTUD; |
| 4449 |
else if (VecVT == MVT::v1i128) |
4449 |
else if (VecVT == MVT::v1i128) |
| 4450 |
return PPC::VCMPGTUQ; |
4450 |
return PPC::VCMPGTUQ; |
| 4451 |
break; |
4451 |
break; |
| 4452 |
default: |
4452 |
default: |
| 4453 |
break; |
4453 |
break; |
| 4454 |
} |
4454 |
} |
| 4455 |
llvm_unreachable("Invalid integer vector compare condition"); |
4455 |
llvm_unreachable("Invalid integer vector compare condition"); |
| 4456 |
} |
4456 |
} |
| 4457 |
} |
4457 |
} |
| 4458 |
|
4458 |
|
| 4459 |
bool PPCDAGToDAGISel::trySETCC(SDNode *N) { |
4459 |
bool PPCDAGToDAGISel::trySETCC(SDNode *N) { |
| 4460 |
SDLoc dl(N); |
4460 |
SDLoc dl(N); |
| 4461 |
unsigned Imm; |
4461 |
unsigned Imm; |
| 4462 |
bool IsStrict = N->isStrictFPOpcode(); |
4462 |
bool IsStrict = N->isStrictFPOpcode(); |
| 4463 |
ISD::CondCode CC = |
4463 |
ISD::CondCode CC = |
| 4464 |
cast(N->getOperand(IsStrict ? 3 : 2))->get(); |
4464 |
cast(N->getOperand(IsStrict ? 3 : 2))->get(); |
| 4465 |
EVT PtrVT = |
4465 |
EVT PtrVT = |
| 4466 |
CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout()); |
4466 |
CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout()); |
| 4467 |
bool isPPC64 = (PtrVT == MVT::i64); |
4467 |
bool isPPC64 = (PtrVT == MVT::i64); |
| 4468 |
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
4468 |
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
| 4469 |
|
4469 |
|
| 4470 |
SDValue LHS = N->getOperand(IsStrict ? 1 : 0); |
4470 |
SDValue LHS = N->getOperand(IsStrict ? 1 : 0); |
| 4471 |
SDValue RHS = N->getOperand(IsStrict ? 2 : 1); |
4471 |
SDValue RHS = N->getOperand(IsStrict ? 2 : 1); |
| 4472 |
|
4472 |
|
| 4473 |
if (!IsStrict && !Subtarget->useCRBits() && isInt32Immediate(RHS, Imm)) { |
4473 |
if (!IsStrict && !Subtarget->useCRBits() && isInt32Immediate(RHS, Imm)) { |
| 4474 |
// We can codegen setcc op, imm very efficiently compared to a brcond. |
4474 |
// We can codegen setcc op, imm very efficiently compared to a brcond. |
| 4475 |
// Check for those cases here. |
4475 |
// Check for those cases here. |
| 4476 |
// setcc op, 0 |
4476 |
// setcc op, 0 |
| 4477 |
if (Imm == 0) { |
4477 |
if (Imm == 0) { |
| 4478 |
SDValue Op = LHS; |
4478 |
SDValue Op = LHS; |
| 4479 |
switch (CC) { |
4479 |
switch (CC) { |
| 4480 |
default: break; |
4480 |
default: break; |
| 4481 |
case ISD::SETEQ: { |
4481 |
case ISD::SETEQ: { |
| 4482 |
Op = SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Op), 0); |
4482 |
Op = SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Op), 0); |
| 4483 |
SDValue Ops[] = { Op, getI32Imm(27, dl), getI32Imm(5, dl), |
4483 |
SDValue Ops[] = { Op, getI32Imm(27, dl), getI32Imm(5, dl), |
| 4484 |
getI32Imm(31, dl) }; |
4484 |
getI32Imm(31, dl) }; |
| 4485 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
4485 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
| 4486 |
return true; |
4486 |
return true; |
| 4487 |
} |
4487 |
} |
| 4488 |
case ISD::SETNE: { |
4488 |
case ISD::SETNE: { |
| 4489 |
if (isPPC64) break; |
4489 |
if (isPPC64) break; |
| 4490 |
SDValue AD = |
4490 |
SDValue AD = |
| 4491 |
SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, |
4491 |
SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, |
| 4492 |
Op, getI32Imm(~0U, dl)), 0); |
4492 |
Op, getI32Imm(~0U, dl)), 0); |
| 4493 |
CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, AD, Op, AD.getValue(1)); |
4493 |
CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, AD, Op, AD.getValue(1)); |
| 4494 |
return true; |
4494 |
return true; |
| 4495 |
} |
4495 |
} |
| 4496 |
case ISD::SETLT: { |
4496 |
case ISD::SETLT: { |
| 4497 |
SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl), |
4497 |
SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl), |
| 4498 |
getI32Imm(31, dl) }; |
4498 |
getI32Imm(31, dl) }; |
| 4499 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
4499 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
| 4500 |
return true; |
4500 |
return true; |
| 4501 |
} |
4501 |
} |
| 4502 |
case ISD::SETGT: { |
4502 |
case ISD::SETGT: { |
| 4503 |
SDValue T = |
4503 |
SDValue T = |
| 4504 |
SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Op), 0); |
4504 |
SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Op), 0); |
| 4505 |
T = SDValue(CurDAG->getMachineNode(PPC::ANDC, dl, MVT::i32, T, Op), 0); |
4505 |
T = SDValue(CurDAG->getMachineNode(PPC::ANDC, dl, MVT::i32, T, Op), 0); |
| 4506 |
SDValue Ops[] = { T, getI32Imm(1, dl), getI32Imm(31, dl), |
4506 |
SDValue Ops[] = { T, getI32Imm(1, dl), getI32Imm(31, dl), |
| 4507 |
getI32Imm(31, dl) }; |
4507 |
getI32Imm(31, dl) }; |
| 4508 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
4508 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
| 4509 |
return true; |
4509 |
return true; |
| 4510 |
} |
4510 |
} |
| 4511 |
} |
4511 |
} |
| 4512 |
} else if (Imm == ~0U) { // setcc op, -1 |
4512 |
} else if (Imm == ~0U) { // setcc op, -1 |
| 4513 |
SDValue Op = LHS; |
4513 |
SDValue Op = LHS; |
| 4514 |
switch (CC) { |
4514 |
switch (CC) { |
| 4515 |
default: break; |
4515 |
default: break; |
| 4516 |
case ISD::SETEQ: |
4516 |
case ISD::SETEQ: |
| 4517 |
if (isPPC64) break; |
4517 |
if (isPPC64) break; |
| 4518 |
Op = SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, |
4518 |
Op = SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, |
| 4519 |
Op, getI32Imm(1, dl)), 0); |
4519 |
Op, getI32Imm(1, dl)), 0); |
| 4520 |
CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, |
4520 |
CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, |
| 4521 |
SDValue(CurDAG->getMachineNode(PPC::LI, dl, |
4521 |
SDValue(CurDAG->getMachineNode(PPC::LI, dl, |
| 4522 |
MVT::i32, |
4522 |
MVT::i32, |
| 4523 |
getI32Imm(0, dl)), |
4523 |
getI32Imm(0, dl)), |
| 4524 |
0), Op.getValue(1)); |
4524 |
0), Op.getValue(1)); |
| 4525 |
return true; |
4525 |
return true; |
| 4526 |
case ISD::SETNE: { |
4526 |
case ISD::SETNE: { |
| 4527 |
if (isPPC64) break; |
4527 |
if (isPPC64) break; |
| 4528 |
Op = SDValue(CurDAG->getMachineNode(PPC::NOR, dl, MVT::i32, Op, Op), 0); |
4528 |
Op = SDValue(CurDAG->getMachineNode(PPC::NOR, dl, MVT::i32, Op, Op), 0); |
| 4529 |
SDNode *AD = CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, |
4529 |
SDNode *AD = CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, |
| 4530 |
Op, getI32Imm(~0U, dl)); |
4530 |
Op, getI32Imm(~0U, dl)); |
| 4531 |
CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(AD, 0), Op, |
4531 |
CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(AD, 0), Op, |
| 4532 |
SDValue(AD, 1)); |
4532 |
SDValue(AD, 1)); |
| 4533 |
return true; |
4533 |
return true; |
| 4534 |
} |
4534 |
} |
| 4535 |
case ISD::SETLT: { |
4535 |
case ISD::SETLT: { |
| 4536 |
SDValue AD = SDValue(CurDAG->getMachineNode(PPC::ADDI, dl, MVT::i32, Op, |
4536 |
SDValue AD = SDValue(CurDAG->getMachineNode(PPC::ADDI, dl, MVT::i32, Op, |
| 4537 |
getI32Imm(1, dl)), 0); |
4537 |
getI32Imm(1, dl)), 0); |
| 4538 |
SDValue AN = SDValue(CurDAG->getMachineNode(PPC::AND, dl, MVT::i32, AD, |
4538 |
SDValue AN = SDValue(CurDAG->getMachineNode(PPC::AND, dl, MVT::i32, AD, |
| 4539 |
Op), 0); |
4539 |
Op), 0); |
| 4540 |
SDValue Ops[] = { AN, getI32Imm(1, dl), getI32Imm(31, dl), |
4540 |
SDValue Ops[] = { AN, getI32Imm(1, dl), getI32Imm(31, dl), |
| 4541 |
getI32Imm(31, dl) }; |
4541 |
getI32Imm(31, dl) }; |
| 4542 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
4542 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
| 4543 |
return true; |
4543 |
return true; |
| 4544 |
} |
4544 |
} |
| 4545 |
case ISD::SETGT: { |
4545 |
case ISD::SETGT: { |
| 4546 |
SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl), |
4546 |
SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl), |
| 4547 |
getI32Imm(31, dl) }; |
4547 |
getI32Imm(31, dl) }; |
| 4548 |
Op = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); |
4548 |
Op = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); |
| 4549 |
CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Op, getI32Imm(1, dl)); |
4549 |
CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Op, getI32Imm(1, dl)); |
| 4550 |
return true; |
4550 |
return true; |
| 4551 |
} |
4551 |
} |
| 4552 |
} |
4552 |
} |
| 4553 |
} |
4553 |
} |
| 4554 |
} |
4554 |
} |
| 4555 |
|
4555 |
|
| 4556 |
// Altivec Vector compare instructions do not set any CR register by default and |
4556 |
// Altivec Vector compare instructions do not set any CR register by default and |
| 4557 |
// vector compare operations return the same type as the operands. |
4557 |
// vector compare operations return the same type as the operands. |
| 4558 |
if (!IsStrict && LHS.getValueType().isVector()) { |
4558 |
if (!IsStrict && LHS.getValueType().isVector()) { |
| 4559 |
if (Subtarget->hasSPE()) |
4559 |
if (Subtarget->hasSPE()) |
| 4560 |
return false; |
4560 |
return false; |
| 4561 |
|
4561 |
|
| 4562 |
EVT VecVT = LHS.getValueType(); |
4562 |
EVT VecVT = LHS.getValueType(); |
| 4563 |
bool Swap, Negate; |
4563 |
bool Swap, Negate; |
| 4564 |
unsigned int VCmpInst = |
4564 |
unsigned int VCmpInst = |
| 4565 |
getVCmpInst(VecVT.getSimpleVT(), CC, Subtarget->hasVSX(), Swap, Negate); |
4565 |
getVCmpInst(VecVT.getSimpleVT(), CC, Subtarget->hasVSX(), Swap, Negate); |
| 4566 |
if (Swap) |
4566 |
if (Swap) |
| 4567 |
std::swap(LHS, RHS); |
4567 |
std::swap(LHS, RHS); |
| 4568 |
|
4568 |
|
| 4569 |
EVT ResVT = VecVT.changeVectorElementTypeToInteger(); |
4569 |
EVT ResVT = VecVT.changeVectorElementTypeToInteger(); |
| 4570 |
if (Negate) { |
4570 |
if (Negate) { |
| 4571 |
SDValue VCmp(CurDAG->getMachineNode(VCmpInst, dl, ResVT, LHS, RHS), 0); |
4571 |
SDValue VCmp(CurDAG->getMachineNode(VCmpInst, dl, ResVT, LHS, RHS), 0); |
| 4572 |
CurDAG->SelectNodeTo(N, Subtarget->hasVSX() ? PPC::XXLNOR : PPC::VNOR, |
4572 |
CurDAG->SelectNodeTo(N, Subtarget->hasVSX() ? PPC::XXLNOR : PPC::VNOR, |
| 4573 |
ResVT, VCmp, VCmp); |
4573 |
ResVT, VCmp, VCmp); |
| 4574 |
return true; |
4574 |
return true; |
| 4575 |
} |
4575 |
} |
| 4576 |
|
4576 |
|
| 4577 |
CurDAG->SelectNodeTo(N, VCmpInst, ResVT, LHS, RHS); |
4577 |
CurDAG->SelectNodeTo(N, VCmpInst, ResVT, LHS, RHS); |
| 4578 |
return true; |
4578 |
return true; |
| 4579 |
} |
4579 |
} |
| 4580 |
|
4580 |
|
| 4581 |
if (Subtarget->useCRBits()) |
4581 |
if (Subtarget->useCRBits()) |
| 4582 |
return false; |
4582 |
return false; |
| 4583 |
|
4583 |
|
| 4584 |
bool Inv; |
4584 |
bool Inv; |
| 4585 |
unsigned Idx = getCRIdxForSetCC(CC, Inv); |
4585 |
unsigned Idx = getCRIdxForSetCC(CC, Inv); |
| 4586 |
SDValue CCReg = SelectCC(LHS, RHS, CC, dl, Chain); |
4586 |
SDValue CCReg = SelectCC(LHS, RHS, CC, dl, Chain); |
| 4587 |
if (IsStrict) |
4587 |
if (IsStrict) |
| 4588 |
CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), CCReg.getValue(1)); |
4588 |
CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), CCReg.getValue(1)); |
| 4589 |
SDValue IntCR; |
4589 |
SDValue IntCR; |
| 4590 |
|
4590 |
|
| 4591 |
// SPE e*cmp* instructions only set the 'gt' bit, so hard-code that |
4591 |
// SPE e*cmp* instructions only set the 'gt' bit, so hard-code that |
| 4592 |
// The correct compare instruction is already set by SelectCC() |
4592 |
// The correct compare instruction is already set by SelectCC() |
| 4593 |
if (Subtarget->hasSPE() && LHS.getValueType().isFloatingPoint()) { |
4593 |
if (Subtarget->hasSPE() && LHS.getValueType().isFloatingPoint()) { |
| 4594 |
Idx = 1; |
4594 |
Idx = 1; |
| 4595 |
} |
4595 |
} |
| 4596 |
|
4596 |
|
| 4597 |
// Force the ccreg into CR7. |
4597 |
// Force the ccreg into CR7. |
| 4598 |
SDValue CR7Reg = CurDAG->getRegister(PPC::CR7, MVT::i32); |
4598 |
SDValue CR7Reg = CurDAG->getRegister(PPC::CR7, MVT::i32); |
| 4599 |
|
4599 |
|
| 4600 |
SDValue InGlue; // Null incoming flag value. |
4600 |
SDValue InGlue; // Null incoming flag value. |
| 4601 |
CCReg = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, CR7Reg, CCReg, |
4601 |
CCReg = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, CR7Reg, CCReg, |
| 4602 |
InGlue).getValue(1); |
4602 |
InGlue).getValue(1); |
| 4603 |
|
4603 |
|
| 4604 |
IntCR = SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR7Reg, |
4604 |
IntCR = SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR7Reg, |
| 4605 |
CCReg), 0); |
4605 |
CCReg), 0); |
| 4606 |
|
4606 |
|
| 4607 |
SDValue Ops[] = { IntCR, getI32Imm((32 - (3 - Idx)) & 31, dl), |
4607 |
SDValue Ops[] = { IntCR, getI32Imm((32 - (3 - Idx)) & 31, dl), |
| 4608 |
getI32Imm(31, dl), getI32Imm(31, dl) }; |
4608 |
getI32Imm(31, dl), getI32Imm(31, dl) }; |
| 4609 |
if (!Inv) { |
4609 |
if (!Inv) { |
| 4610 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
4610 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
| 4611 |
return true; |
4611 |
return true; |
| 4612 |
} |
4612 |
} |
| 4613 |
|
4613 |
|
| 4614 |
// Get the specified bit. |
4614 |
// Get the specified bit. |
| 4615 |
SDValue Tmp = |
4615 |
SDValue Tmp = |
| 4616 |
SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); |
4616 |
SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); |
| 4617 |
CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Tmp, getI32Imm(1, dl)); |
4617 |
CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Tmp, getI32Imm(1, dl)); |
| 4618 |
return true; |
4618 |
return true; |
| 4619 |
} |
4619 |
} |
| 4620 |
|
4620 |
|
| 4621 |
/// Does this node represent a load/store node whose address can be represented |
4621 |
/// Does this node represent a load/store node whose address can be represented |
| 4622 |
/// with a register plus an immediate that's a multiple of \p Val: |
4622 |
/// with a register plus an immediate that's a multiple of \p Val: |
| 4623 |
bool PPCDAGToDAGISel::isOffsetMultipleOf(SDNode *N, unsigned Val) const { |
4623 |
bool PPCDAGToDAGISel::isOffsetMultipleOf(SDNode *N, unsigned Val) const { |
| 4624 |
LoadSDNode *LDN = dyn_cast(N); |
4624 |
LoadSDNode *LDN = dyn_cast(N); |
| 4625 |
StoreSDNode *STN = dyn_cast(N); |
4625 |
StoreSDNode *STN = dyn_cast(N); |
| 4626 |
MemIntrinsicSDNode *MIN = dyn_cast(N); |
4626 |
MemIntrinsicSDNode *MIN = dyn_cast(N); |
| 4627 |
SDValue AddrOp; |
4627 |
SDValue AddrOp; |
| 4628 |
if (LDN || (MIN && MIN->getOpcode() == PPCISD::LD_SPLAT)) |
4628 |
if (LDN || (MIN && MIN->getOpcode() == PPCISD::LD_SPLAT)) |
| 4629 |
AddrOp = N->getOperand(1); |
4629 |
AddrOp = N->getOperand(1); |
| 4630 |
else if (STN) |
4630 |
else if (STN) |
| 4631 |
AddrOp = STN->getOperand(2); |
4631 |
AddrOp = STN->getOperand(2); |
| 4632 |
|
4632 |
|
| 4633 |
// If the address points a frame object or a frame object with an offset, |
4633 |
// If the address points a frame object or a frame object with an offset, |
| 4634 |
// we need to check the object alignment. |
4634 |
// we need to check the object alignment. |
| 4635 |
short Imm = 0; |
4635 |
short Imm = 0; |
| 4636 |
if (FrameIndexSDNode *FI = dyn_cast( |
4636 |
if (FrameIndexSDNode *FI = dyn_cast( |
| 4637 |
AddrOp.getOpcode() == ISD::ADD ? AddrOp.getOperand(0) : |
4637 |
AddrOp.getOpcode() == ISD::ADD ? AddrOp.getOperand(0) : |
| 4638 |
AddrOp)) { |
4638 |
AddrOp)) { |
| 4639 |
// If op0 is a frame index that is under aligned, we can't do it either, |
4639 |
// If op0 is a frame index that is under aligned, we can't do it either, |
| 4640 |
// because it is translated to r31 or r1 + slot + offset. We won't know the |
4640 |
// because it is translated to r31 or r1 + slot + offset. We won't know the |
| 4641 |
// slot number until the stack frame is finalized. |
4641 |
// slot number until the stack frame is finalized. |
| 4642 |
const MachineFrameInfo &MFI = CurDAG->getMachineFunction().getFrameInfo(); |
4642 |
const MachineFrameInfo &MFI = CurDAG->getMachineFunction().getFrameInfo(); |
| 4643 |
unsigned SlotAlign = MFI.getObjectAlign(FI->getIndex()).value(); |
4643 |
unsigned SlotAlign = MFI.getObjectAlign(FI->getIndex()).value(); |
| 4644 |
if ((SlotAlign % Val) != 0) |
4644 |
if ((SlotAlign % Val) != 0) |
| 4645 |
return false; |
4645 |
return false; |
| 4646 |
|
4646 |
|
| 4647 |
// If we have an offset, we need further check on the offset. |
4647 |
// If we have an offset, we need further check on the offset. |
| 4648 |
if (AddrOp.getOpcode() != ISD::ADD) |
4648 |
if (AddrOp.getOpcode() != ISD::ADD) |
| 4649 |
return true; |
4649 |
return true; |
| 4650 |
} |
4650 |
} |
| 4651 |
|
4651 |
|
| 4652 |
if (AddrOp.getOpcode() == ISD::ADD) |
4652 |
if (AddrOp.getOpcode() == ISD::ADD) |
| 4653 |
return isIntS16Immediate(AddrOp.getOperand(1), Imm) && !(Imm % Val); |
4653 |
return isIntS16Immediate(AddrOp.getOperand(1), Imm) && !(Imm % Val); |
| 4654 |
|
4654 |
|
| 4655 |
// If the address comes from the outside, the offset will be zero. |
4655 |
// If the address comes from the outside, the offset will be zero. |
| 4656 |
return AddrOp.getOpcode() == ISD::CopyFromReg; |
4656 |
return AddrOp.getOpcode() == ISD::CopyFromReg; |
| 4657 |
} |
4657 |
} |
| 4658 |
|
4658 |
|
| 4659 |
void PPCDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) { |
4659 |
void PPCDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) { |
| 4660 |
// Transfer memoperands. |
4660 |
// Transfer memoperands. |
| 4661 |
MachineMemOperand *MemOp = cast(N)->getMemOperand(); |
4661 |
MachineMemOperand *MemOp = cast(N)->getMemOperand(); |
| 4662 |
CurDAG->setNodeMemRefs(cast(Result), {MemOp}); |
4662 |
CurDAG->setNodeMemRefs(cast(Result), {MemOp}); |
| 4663 |
} |
4663 |
} |
| 4664 |
|
4664 |
|
| 4665 |
static bool mayUseP9Setb(SDNode *N, const ISD::CondCode &CC, SelectionDAG *DAG, |
4665 |
static bool mayUseP9Setb(SDNode *N, const ISD::CondCode &CC, SelectionDAG *DAG, |
| 4666 |
bool &NeedSwapOps, bool &IsUnCmp) { |
4666 |
bool &NeedSwapOps, bool &IsUnCmp) { |
| 4667 |
|
4667 |
|
| 4668 |
assert(N->getOpcode() == ISD::SELECT_CC && "Expecting a SELECT_CC here."); |
4668 |
assert(N->getOpcode() == ISD::SELECT_CC && "Expecting a SELECT_CC here."); |
| 4669 |
|
4669 |
|
| 4670 |
SDValue LHS = N->getOperand(0); |
4670 |
SDValue LHS = N->getOperand(0); |
| 4671 |
SDValue RHS = N->getOperand(1); |
4671 |
SDValue RHS = N->getOperand(1); |
| 4672 |
SDValue TrueRes = N->getOperand(2); |
4672 |
SDValue TrueRes = N->getOperand(2); |
| 4673 |
SDValue FalseRes = N->getOperand(3); |
4673 |
SDValue FalseRes = N->getOperand(3); |
| 4674 |
ConstantSDNode *TrueConst = dyn_cast(TrueRes); |
4674 |
ConstantSDNode *TrueConst = dyn_cast(TrueRes); |
| 4675 |
if (!TrueConst || (N->getSimpleValueType(0) != MVT::i64 && |
4675 |
if (!TrueConst || (N->getSimpleValueType(0) != MVT::i64 && |
| 4676 |
N->getSimpleValueType(0) != MVT::i32)) |
4676 |
N->getSimpleValueType(0) != MVT::i32)) |
| 4677 |
return false; |
4677 |
return false; |
| 4678 |
|
4678 |
|
| 4679 |
// We are looking for any of: |
4679 |
// We are looking for any of: |
| 4680 |
// (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, cc2)), cc1) |
4680 |
// (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, cc2)), cc1) |
| 4681 |
// (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, cc2)), cc1) |
4681 |
// (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, cc2)), cc1) |
| 4682 |
// (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, 1, -1, cc2), seteq) |
4682 |
// (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, 1, -1, cc2), seteq) |
| 4683 |
// (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, -1, 1, cc2), seteq) |
4683 |
// (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, -1, 1, cc2), seteq) |
| 4684 |
int64_t TrueResVal = TrueConst->getSExtValue(); |
4684 |
int64_t TrueResVal = TrueConst->getSExtValue(); |
| 4685 |
if ((TrueResVal < -1 || TrueResVal > 1) || |
4685 |
if ((TrueResVal < -1 || TrueResVal > 1) || |
| 4686 |
(TrueResVal == -1 && FalseRes.getOpcode() != ISD::ZERO_EXTEND) || |
4686 |
(TrueResVal == -1 && FalseRes.getOpcode() != ISD::ZERO_EXTEND) || |
| 4687 |
(TrueResVal == 1 && FalseRes.getOpcode() != ISD::SIGN_EXTEND) || |
4687 |
(TrueResVal == 1 && FalseRes.getOpcode() != ISD::SIGN_EXTEND) || |
| 4688 |
(TrueResVal == 0 && |
4688 |
(TrueResVal == 0 && |
| 4689 |
(FalseRes.getOpcode() != ISD::SELECT_CC || CC != ISD::SETEQ))) |
4689 |
(FalseRes.getOpcode() != ISD::SELECT_CC || CC != ISD::SETEQ))) |
| 4690 |
return false; |
4690 |
return false; |
| 4691 |
|
4691 |
|
| 4692 |
SDValue SetOrSelCC = FalseRes.getOpcode() == ISD::SELECT_CC |
4692 |
SDValue SetOrSelCC = FalseRes.getOpcode() == ISD::SELECT_CC |
| 4693 |
? FalseRes |
4693 |
? FalseRes |
| 4694 |
: FalseRes.getOperand(0); |
4694 |
: FalseRes.getOperand(0); |
| 4695 |
bool InnerIsSel = SetOrSelCC.getOpcode() == ISD::SELECT_CC; |
4695 |
bool InnerIsSel = SetOrSelCC.getOpcode() == ISD::SELECT_CC; |
| 4696 |
if (SetOrSelCC.getOpcode() != ISD::SETCC && |
4696 |
if (SetOrSelCC.getOpcode() != ISD::SETCC && |
| 4697 |
SetOrSelCC.getOpcode() != ISD::SELECT_CC) |
4697 |
SetOrSelCC.getOpcode() != ISD::SELECT_CC) |
| 4698 |
return false; |
4698 |
return false; |
| 4699 |
|
4699 |
|
| 4700 |
// Without this setb optimization, the outer SELECT_CC will be manually |
4700 |
// Without this setb optimization, the outer SELECT_CC will be manually |
| 4701 |
// selected to SELECT_CC_I4/SELECT_CC_I8 Pseudo, then expand-isel-pseudos pass |
4701 |
// selected to SELECT_CC_I4/SELECT_CC_I8 Pseudo, then expand-isel-pseudos pass |
| 4702 |
// transforms pseudo instruction to isel instruction. When there are more than |
4702 |
// transforms pseudo instruction to isel instruction. When there are more than |
| 4703 |
// one use for result like zext/sext, with current optimization we only see |
4703 |
// one use for result like zext/sext, with current optimization we only see |
| 4704 |
// isel is replaced by setb but can't see any significant gain. Since |
4704 |
// isel is replaced by setb but can't see any significant gain. Since |
| 4705 |
// setb has longer latency than original isel, we should avoid this. Another |
4705 |
// setb has longer latency than original isel, we should avoid this. Another |
| 4706 |
// point is that setb requires comparison always kept, it can break the |
4706 |
// point is that setb requires comparison always kept, it can break the |
| 4707 |
// opportunity to get the comparison away if we have in future. |
4707 |
// opportunity to get the comparison away if we have in future. |
| 4708 |
if (!SetOrSelCC.hasOneUse() || (!InnerIsSel && !FalseRes.hasOneUse())) |
4708 |
if (!SetOrSelCC.hasOneUse() || (!InnerIsSel && !FalseRes.hasOneUse())) |
| 4709 |
return false; |
4709 |
return false; |
| 4710 |
|
4710 |
|
| 4711 |
SDValue InnerLHS = SetOrSelCC.getOperand(0); |
4711 |
SDValue InnerLHS = SetOrSelCC.getOperand(0); |
| 4712 |
SDValue InnerRHS = SetOrSelCC.getOperand(1); |
4712 |
SDValue InnerRHS = SetOrSelCC.getOperand(1); |
| 4713 |
ISD::CondCode InnerCC = |
4713 |
ISD::CondCode InnerCC = |
| 4714 |
cast(SetOrSelCC.getOperand(InnerIsSel ? 4 : 2))->get(); |
4714 |
cast(SetOrSelCC.getOperand(InnerIsSel ? 4 : 2))->get(); |
| 4715 |
// If the inner comparison is a select_cc, make sure the true/false values are |
4715 |
// If the inner comparison is a select_cc, make sure the true/false values are |
| 4716 |
// 1/-1 and canonicalize it if needed. |
4716 |
// 1/-1 and canonicalize it if needed. |
| 4717 |
if (InnerIsSel) { |
4717 |
if (InnerIsSel) { |
| 4718 |
ConstantSDNode *SelCCTrueConst = |
4718 |
ConstantSDNode *SelCCTrueConst = |
| 4719 |
dyn_cast(SetOrSelCC.getOperand(2)); |
4719 |
dyn_cast(SetOrSelCC.getOperand(2)); |
| 4720 |
ConstantSDNode *SelCCFalseConst = |
4720 |
ConstantSDNode *SelCCFalseConst = |
| 4721 |
dyn_cast(SetOrSelCC.getOperand(3)); |
4721 |
dyn_cast(SetOrSelCC.getOperand(3)); |
| 4722 |
if (!SelCCTrueConst || !SelCCFalseConst) |
4722 |
if (!SelCCTrueConst || !SelCCFalseConst) |
| 4723 |
return false; |
4723 |
return false; |
| 4724 |
int64_t SelCCTVal = SelCCTrueConst->getSExtValue(); |
4724 |
int64_t SelCCTVal = SelCCTrueConst->getSExtValue(); |
| 4725 |
int64_t SelCCFVal = SelCCFalseConst->getSExtValue(); |
4725 |
int64_t SelCCFVal = SelCCFalseConst->getSExtValue(); |
| 4726 |
// The values must be -1/1 (requiring a swap) or 1/-1. |
4726 |
// The values must be -1/1 (requiring a swap) or 1/-1. |
| 4727 |
if (SelCCTVal == -1 && SelCCFVal == 1) { |
4727 |
if (SelCCTVal == -1 && SelCCFVal == 1) { |
| 4728 |
std::swap(InnerLHS, InnerRHS); |
4728 |
std::swap(InnerLHS, InnerRHS); |
| 4729 |
} else if (SelCCTVal != 1 || SelCCFVal != -1) |
4729 |
} else if (SelCCTVal != 1 || SelCCFVal != -1) |
| 4730 |
return false; |
4730 |
return false; |
| 4731 |
} |
4731 |
} |
| 4732 |
|
4732 |
|
| 4733 |
// Canonicalize unsigned case |
4733 |
// Canonicalize unsigned case |
| 4734 |
if (InnerCC == ISD::SETULT || InnerCC == ISD::SETUGT) { |
4734 |
if (InnerCC == ISD::SETULT || InnerCC == ISD::SETUGT) { |
| 4735 |
IsUnCmp = true; |
4735 |
IsUnCmp = true; |
| 4736 |
InnerCC = (InnerCC == ISD::SETULT) ? ISD::SETLT : ISD::SETGT; |
4736 |
InnerCC = (InnerCC == ISD::SETULT) ? ISD::SETLT : ISD::SETGT; |
| 4737 |
} |
4737 |
} |
| 4738 |
|
4738 |
|
| 4739 |
bool InnerSwapped = false; |
4739 |
bool InnerSwapped = false; |
| 4740 |
if (LHS == InnerRHS && RHS == InnerLHS) |
4740 |
if (LHS == InnerRHS && RHS == InnerLHS) |
| 4741 |
InnerSwapped = true; |
4741 |
InnerSwapped = true; |
| 4742 |
else if (LHS != InnerLHS || RHS != InnerRHS) |
4742 |
else if (LHS != InnerLHS || RHS != InnerRHS) |
| 4743 |
return false; |
4743 |
return false; |
| 4744 |
|
4744 |
|
| 4745 |
switch (CC) { |
4745 |
switch (CC) { |
| 4746 |
// (select_cc lhs, rhs, 0, \ |
4746 |
// (select_cc lhs, rhs, 0, \ |
| 4747 |
// (select_cc [lr]hs, [lr]hs, 1, -1, setlt/setgt), seteq) |
4747 |
// (select_cc [lr]hs, [lr]hs, 1, -1, setlt/setgt), seteq) |
| 4748 |
case ISD::SETEQ: |
4748 |
case ISD::SETEQ: |
| 4749 |
if (!InnerIsSel) |
4749 |
if (!InnerIsSel) |
| 4750 |
return false; |
4750 |
return false; |
| 4751 |
if (InnerCC != ISD::SETLT && InnerCC != ISD::SETGT) |
4751 |
if (InnerCC != ISD::SETLT && InnerCC != ISD::SETGT) |
| 4752 |
return false; |
4752 |
return false; |
| 4753 |
NeedSwapOps = (InnerCC == ISD::SETGT) ? InnerSwapped : !InnerSwapped; |
4753 |
NeedSwapOps = (InnerCC == ISD::SETGT) ? InnerSwapped : !InnerSwapped; |
| 4754 |
break; |
4754 |
break; |
| 4755 |
|
4755 |
|
| 4756 |
// (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?lt) |
4756 |
// (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?lt) |
| 4757 |
// (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setgt)), setu?lt) |
4757 |
// (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setgt)), setu?lt) |
| 4758 |
// (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setlt)), setu?lt) |
4758 |
// (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setlt)), setu?lt) |
| 4759 |
// (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?lt) |
4759 |
// (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?lt) |
| 4760 |
// (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setgt)), setu?lt) |
4760 |
// (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setgt)), setu?lt) |
| 4761 |
// (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setlt)), setu?lt) |
4761 |
// (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setlt)), setu?lt) |
| 4762 |
case ISD::SETULT: |
4762 |
case ISD::SETULT: |
| 4763 |
if (!IsUnCmp && InnerCC != ISD::SETNE) |
4763 |
if (!IsUnCmp && InnerCC != ISD::SETNE) |
| 4764 |
return false; |
4764 |
return false; |
| 4765 |
IsUnCmp = true; |
4765 |
IsUnCmp = true; |
| 4766 |
[[fallthrough]]; |
4766 |
[[fallthrough]]; |
| 4767 |
case ISD::SETLT: |
4767 |
case ISD::SETLT: |
| 4768 |
if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETGT && !InnerSwapped) || |
4768 |
if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETGT && !InnerSwapped) || |
| 4769 |
(InnerCC == ISD::SETLT && InnerSwapped)) |
4769 |
(InnerCC == ISD::SETLT && InnerSwapped)) |
| 4770 |
NeedSwapOps = (TrueResVal == 1); |
4770 |
NeedSwapOps = (TrueResVal == 1); |
| 4771 |
else |
4771 |
else |
| 4772 |
return false; |
4772 |
return false; |
| 4773 |
break; |
4773 |
break; |
| 4774 |
|
4774 |
|
| 4775 |
// (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?gt) |
4775 |
// (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?gt) |
| 4776 |
// (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setlt)), setu?gt) |
4776 |
// (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setlt)), setu?gt) |
| 4777 |
// (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setgt)), setu?gt) |
4777 |
// (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setgt)), setu?gt) |
| 4778 |
// (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?gt) |
4778 |
// (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?gt) |
| 4779 |
// (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setlt)), setu?gt) |
4779 |
// (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setlt)), setu?gt) |
| 4780 |
// (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setgt)), setu?gt) |
4780 |
// (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setgt)), setu?gt) |
| 4781 |
case ISD::SETUGT: |
4781 |
case ISD::SETUGT: |
| 4782 |
if (!IsUnCmp && InnerCC != ISD::SETNE) |
4782 |
if (!IsUnCmp && InnerCC != ISD::SETNE) |
| 4783 |
return false; |
4783 |
return false; |
| 4784 |
IsUnCmp = true; |
4784 |
IsUnCmp = true; |
| 4785 |
[[fallthrough]]; |
4785 |
[[fallthrough]]; |
| 4786 |
case ISD::SETGT: |
4786 |
case ISD::SETGT: |
| 4787 |
if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETLT && !InnerSwapped) || |
4787 |
if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETLT && !InnerSwapped) || |
| 4788 |
(InnerCC == ISD::SETGT && InnerSwapped)) |
4788 |
(InnerCC == ISD::SETGT && InnerSwapped)) |
| 4789 |
NeedSwapOps = (TrueResVal == -1); |
4789 |
NeedSwapOps = (TrueResVal == -1); |
| 4790 |
else |
4790 |
else |
| 4791 |
return false; |
4791 |
return false; |
| 4792 |
break; |
4792 |
break; |
| 4793 |
|
4793 |
|
| 4794 |
default: |
4794 |
default: |
| 4795 |
return false; |
4795 |
return false; |
| 4796 |
} |
4796 |
} |
| 4797 |
|
4797 |
|
| 4798 |
LLVM_DEBUG(dbgs() << "Found a node that can be lowered to a SETB: "); |
4798 |
LLVM_DEBUG(dbgs() << "Found a node that can be lowered to a SETB: "); |
| 4799 |
LLVM_DEBUG(N->dump()); |
4799 |
LLVM_DEBUG(N->dump()); |
| 4800 |
|
4800 |
|
| 4801 |
return true; |
4801 |
return true; |
| 4802 |
} |
4802 |
} |
| 4803 |
|
4803 |
|
| 4804 |
// Return true if it's a software square-root/divide operand. |
4804 |
// Return true if it's a software square-root/divide operand. |
| 4805 |
static bool isSWTestOp(SDValue N) { |
4805 |
static bool isSWTestOp(SDValue N) { |
| 4806 |
if (N.getOpcode() == PPCISD::FTSQRT) |
4806 |
if (N.getOpcode() == PPCISD::FTSQRT) |
| 4807 |
return true; |
4807 |
return true; |
| 4808 |
if (N.getNumOperands() < 1 || !isa(N.getOperand(0)) || |
4808 |
if (N.getNumOperands() < 1 || !isa(N.getOperand(0)) || |
| 4809 |
N.getOpcode() != ISD::INTRINSIC_WO_CHAIN) |
4809 |
N.getOpcode() != ISD::INTRINSIC_WO_CHAIN) |
| 4810 |
return false; |
4810 |
return false; |
| 4811 |
switch (N.getConstantOperandVal(0)) { |
4811 |
switch (N.getConstantOperandVal(0)) { |
| 4812 |
case Intrinsic::ppc_vsx_xvtdivdp: |
4812 |
case Intrinsic::ppc_vsx_xvtdivdp: |
| 4813 |
case Intrinsic::ppc_vsx_xvtdivsp: |
4813 |
case Intrinsic::ppc_vsx_xvtdivsp: |
| 4814 |
case Intrinsic::ppc_vsx_xvtsqrtdp: |
4814 |
case Intrinsic::ppc_vsx_xvtsqrtdp: |
| 4815 |
case Intrinsic::ppc_vsx_xvtsqrtsp: |
4815 |
case Intrinsic::ppc_vsx_xvtsqrtsp: |
| 4816 |
return true; |
4816 |
return true; |
| 4817 |
} |
4817 |
} |
| 4818 |
return false; |
4818 |
return false; |
| 4819 |
} |
4819 |
} |
| 4820 |
|
4820 |
|
| 4821 |
bool PPCDAGToDAGISel::tryFoldSWTestBRCC(SDNode *N) { |
4821 |
bool PPCDAGToDAGISel::tryFoldSWTestBRCC(SDNode *N) { |
| 4822 |
assert(N->getOpcode() == ISD::BR_CC && "ISD::BR_CC is expected."); |
4822 |
assert(N->getOpcode() == ISD::BR_CC && "ISD::BR_CC is expected."); |
| 4823 |
// We are looking for following patterns, where `truncate to i1` actually has |
4823 |
// We are looking for following patterns, where `truncate to i1` actually has |
| 4824 |
// the same semantic with `and 1`. |
4824 |
// the same semantic with `and 1`. |
| 4825 |
// (br_cc seteq, (truncateToi1 SWTestOp), 0) -> (BCC PRED_NU, SWTestOp) |
4825 |
// (br_cc seteq, (truncateToi1 SWTestOp), 0) -> (BCC PRED_NU, SWTestOp) |
| 4826 |
// (br_cc seteq, (and SWTestOp, 2), 0) -> (BCC PRED_NE, SWTestOp) |
4826 |
// (br_cc seteq, (and SWTestOp, 2), 0) -> (BCC PRED_NE, SWTestOp) |
| 4827 |
// (br_cc seteq, (and SWTestOp, 4), 0) -> (BCC PRED_LE, SWTestOp) |
4827 |
// (br_cc seteq, (and SWTestOp, 4), 0) -> (BCC PRED_LE, SWTestOp) |
| 4828 |
// (br_cc seteq, (and SWTestOp, 8), 0) -> (BCC PRED_GE, SWTestOp) |
4828 |
// (br_cc seteq, (and SWTestOp, 8), 0) -> (BCC PRED_GE, SWTestOp) |
| 4829 |
// (br_cc setne, (truncateToi1 SWTestOp), 0) -> (BCC PRED_UN, SWTestOp) |
4829 |
// (br_cc setne, (truncateToi1 SWTestOp), 0) -> (BCC PRED_UN, SWTestOp) |
| 4830 |
// (br_cc setne, (and SWTestOp, 2), 0) -> (BCC PRED_EQ, SWTestOp) |
4830 |
// (br_cc setne, (and SWTestOp, 2), 0) -> (BCC PRED_EQ, SWTestOp) |
| 4831 |
// (br_cc setne, (and SWTestOp, 4), 0) -> (BCC PRED_GT, SWTestOp) |
4831 |
// (br_cc setne, (and SWTestOp, 4), 0) -> (BCC PRED_GT, SWTestOp) |
| 4832 |
// (br_cc setne, (and SWTestOp, 8), 0) -> (BCC PRED_LT, SWTestOp) |
4832 |
// (br_cc setne, (and SWTestOp, 8), 0) -> (BCC PRED_LT, SWTestOp) |
| 4833 |
ISD::CondCode CC = cast(N->getOperand(1))->get(); |
4833 |
ISD::CondCode CC = cast(N->getOperand(1))->get(); |
| 4834 |
if (CC != ISD::SETEQ && CC != ISD::SETNE) |
4834 |
if (CC != ISD::SETEQ && CC != ISD::SETNE) |
| 4835 |
return false; |
4835 |
return false; |
| 4836 |
|
4836 |
|
| 4837 |
SDValue CmpRHS = N->getOperand(3); |
4837 |
SDValue CmpRHS = N->getOperand(3); |
| 4838 |
if (!isa(CmpRHS) || |
4838 |
if (!isa(CmpRHS) || |
| 4839 |
cast(CmpRHS)->getSExtValue() != 0) |
4839 |
cast(CmpRHS)->getSExtValue() != 0) |
| 4840 |
return false; |
4840 |
return false; |
| 4841 |
|
4841 |
|
| 4842 |
SDValue CmpLHS = N->getOperand(2); |
4842 |
SDValue CmpLHS = N->getOperand(2); |
| 4843 |
if (CmpLHS.getNumOperands() < 1 || !isSWTestOp(CmpLHS.getOperand(0))) |
4843 |
if (CmpLHS.getNumOperands() < 1 || !isSWTestOp(CmpLHS.getOperand(0))) |
| 4844 |
return false; |
4844 |
return false; |
| 4845 |
|
4845 |
|
| 4846 |
unsigned PCC = 0; |
4846 |
unsigned PCC = 0; |
| 4847 |
bool IsCCNE = CC == ISD::SETNE; |
4847 |
bool IsCCNE = CC == ISD::SETNE; |
| 4848 |
if (CmpLHS.getOpcode() == ISD::AND && |
4848 |
if (CmpLHS.getOpcode() == ISD::AND && |
| 4849 |
isa(CmpLHS.getOperand(1))) |
4849 |
isa(CmpLHS.getOperand(1))) |
| 4850 |
switch (CmpLHS.getConstantOperandVal(1)) { |
4850 |
switch (CmpLHS.getConstantOperandVal(1)) { |
| 4851 |
case 1: |
4851 |
case 1: |
| 4852 |
PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU; |
4852 |
PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU; |
| 4853 |
break; |
4853 |
break; |
| 4854 |
case 2: |
4854 |
case 2: |
| 4855 |
PCC = IsCCNE ? PPC::PRED_EQ : PPC::PRED_NE; |
4855 |
PCC = IsCCNE ? PPC::PRED_EQ : PPC::PRED_NE; |
| 4856 |
break; |
4856 |
break; |
| 4857 |
case 4: |
4857 |
case 4: |
| 4858 |
PCC = IsCCNE ? PPC::PRED_GT : PPC::PRED_LE; |
4858 |
PCC = IsCCNE ? PPC::PRED_GT : PPC::PRED_LE; |
| 4859 |
break; |
4859 |
break; |
| 4860 |
case 8: |
4860 |
case 8: |
| 4861 |
PCC = IsCCNE ? PPC::PRED_LT : PPC::PRED_GE; |
4861 |
PCC = IsCCNE ? PPC::PRED_LT : PPC::PRED_GE; |
| 4862 |
break; |
4862 |
break; |
| 4863 |
default: |
4863 |
default: |
| 4864 |
return false; |
4864 |
return false; |
| 4865 |
} |
4865 |
} |
| 4866 |
else if (CmpLHS.getOpcode() == ISD::TRUNCATE && |
4866 |
else if (CmpLHS.getOpcode() == ISD::TRUNCATE && |
| 4867 |
CmpLHS.getValueType() == MVT::i1) |
4867 |
CmpLHS.getValueType() == MVT::i1) |
| 4868 |
PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU; |
4868 |
PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU; |
| 4869 |
|
4869 |
|
| 4870 |
if (PCC) { |
4870 |
if (PCC) { |
| 4871 |
SDLoc dl(N); |
4871 |
SDLoc dl(N); |
| 4872 |
SDValue Ops[] = {getI32Imm(PCC, dl), CmpLHS.getOperand(0), N->getOperand(4), |
4872 |
SDValue Ops[] = {getI32Imm(PCC, dl), CmpLHS.getOperand(0), N->getOperand(4), |
| 4873 |
N->getOperand(0)}; |
4873 |
N->getOperand(0)}; |
| 4874 |
CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); |
4874 |
CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); |
| 4875 |
return true; |
4875 |
return true; |
| 4876 |
} |
4876 |
} |
| 4877 |
return false; |
4877 |
return false; |
| 4878 |
} |
4878 |
} |
| 4879 |
|
4879 |
|
| 4880 |
bool PPCDAGToDAGISel::trySelectLoopCountIntrinsic(SDNode *N) { |
4880 |
bool PPCDAGToDAGISel::trySelectLoopCountIntrinsic(SDNode *N) { |
| 4881 |
// Sometimes the promoted value of the intrinsic is ANDed by some non-zero |
4881 |
// Sometimes the promoted value of the intrinsic is ANDed by some non-zero |
| 4882 |
// value, for example when crbits is disabled. If so, select the |
4882 |
// value, for example when crbits is disabled. If so, select the |
| 4883 |
// loop_decrement intrinsics now. |
4883 |
// loop_decrement intrinsics now. |
| 4884 |
ISD::CondCode CC = cast(N->getOperand(1))->get(); |
4884 |
ISD::CondCode CC = cast(N->getOperand(1))->get(); |
| 4885 |
SDValue LHS = N->getOperand(2), RHS = N->getOperand(3); |
4885 |
SDValue LHS = N->getOperand(2), RHS = N->getOperand(3); |
| 4886 |
|
4886 |
|
| 4887 |
if (LHS.getOpcode() != ISD::AND || !isa(LHS.getOperand(1)) || |
4887 |
if (LHS.getOpcode() != ISD::AND || !isa(LHS.getOperand(1)) || |
| 4888 |
isNullConstant(LHS.getOperand(1))) |
4888 |
isNullConstant(LHS.getOperand(1))) |
| 4889 |
return false; |
4889 |
return false; |
| 4890 |
|
4890 |
|
| 4891 |
if (LHS.getOperand(0).getOpcode() != ISD::INTRINSIC_W_CHAIN || |
4891 |
if (LHS.getOperand(0).getOpcode() != ISD::INTRINSIC_W_CHAIN || |
| 4892 |
cast(LHS.getOperand(0).getOperand(1))->getZExtValue() != |
4892 |
cast(LHS.getOperand(0).getOperand(1))->getZExtValue() != |
| 4893 |
Intrinsic::loop_decrement) |
4893 |
Intrinsic::loop_decrement) |
| 4894 |
return false; |
4894 |
return false; |
| 4895 |
|
4895 |
|
| 4896 |
if (!isa(RHS)) |
4896 |
if (!isa(RHS)) |
| 4897 |
return false; |
4897 |
return false; |
| 4898 |
|
4898 |
|
| 4899 |
assert((CC == ISD::SETEQ || CC == ISD::SETNE) && |
4899 |
assert((CC == ISD::SETEQ || CC == ISD::SETNE) && |
| 4900 |
"Counter decrement comparison is not EQ or NE"); |
4900 |
"Counter decrement comparison is not EQ or NE"); |
| 4901 |
|
4901 |
|
| 4902 |
SDValue OldDecrement = LHS.getOperand(0); |
4902 |
SDValue OldDecrement = LHS.getOperand(0); |
| 4903 |
assert(OldDecrement.hasOneUse() && "loop decrement has more than one use!"); |
4903 |
assert(OldDecrement.hasOneUse() && "loop decrement has more than one use!"); |
| 4904 |
|
4904 |
|
| 4905 |
SDLoc DecrementLoc(OldDecrement); |
4905 |
SDLoc DecrementLoc(OldDecrement); |
| 4906 |
SDValue ChainInput = OldDecrement.getOperand(0); |
4906 |
SDValue ChainInput = OldDecrement.getOperand(0); |
| 4907 |
SDValue DecrementOps[] = {Subtarget->isPPC64() ? getI64Imm(1, DecrementLoc) |
4907 |
SDValue DecrementOps[] = {Subtarget->isPPC64() ? getI64Imm(1, DecrementLoc) |
| 4908 |
: getI32Imm(1, DecrementLoc)}; |
4908 |
: getI32Imm(1, DecrementLoc)}; |
| 4909 |
unsigned DecrementOpcode = |
4909 |
unsigned DecrementOpcode = |
| 4910 |
Subtarget->isPPC64() ? PPC::DecreaseCTR8loop : PPC::DecreaseCTRloop; |
4910 |
Subtarget->isPPC64() ? PPC::DecreaseCTR8loop : PPC::DecreaseCTRloop; |
| 4911 |
SDNode *NewDecrement = CurDAG->getMachineNode(DecrementOpcode, DecrementLoc, |
4911 |
SDNode *NewDecrement = CurDAG->getMachineNode(DecrementOpcode, DecrementLoc, |
| 4912 |
MVT::i1, DecrementOps); |
4912 |
MVT::i1, DecrementOps); |
| 4913 |
|
4913 |
|
| 4914 |
unsigned Val = cast(RHS)->getZExtValue(); |
4914 |
unsigned Val = cast(RHS)->getZExtValue(); |
| 4915 |
bool IsBranchOnTrue = (CC == ISD::SETEQ && Val) || (CC == ISD::SETNE && !Val); |
4915 |
bool IsBranchOnTrue = (CC == ISD::SETEQ && Val) || (CC == ISD::SETNE && !Val); |
| 4916 |
unsigned Opcode = IsBranchOnTrue ? PPC::BC : PPC::BCn; |
4916 |
unsigned Opcode = IsBranchOnTrue ? PPC::BC : PPC::BCn; |
| 4917 |
|
4917 |
|
| 4918 |
ReplaceUses(LHS.getValue(0), LHS.getOperand(1)); |
4918 |
ReplaceUses(LHS.getValue(0), LHS.getOperand(1)); |
| 4919 |
CurDAG->RemoveDeadNode(LHS.getNode()); |
4919 |
CurDAG->RemoveDeadNode(LHS.getNode()); |
| 4920 |
|
4920 |
|
| 4921 |
// Mark the old loop_decrement intrinsic as dead. |
4921 |
// Mark the old loop_decrement intrinsic as dead. |
| 4922 |
ReplaceUses(OldDecrement.getValue(1), ChainInput); |
4922 |
ReplaceUses(OldDecrement.getValue(1), ChainInput); |
| 4923 |
CurDAG->RemoveDeadNode(OldDecrement.getNode()); |
4923 |
CurDAG->RemoveDeadNode(OldDecrement.getNode()); |
| 4924 |
|
4924 |
|
| 4925 |
SDValue Chain = CurDAG->getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, |
4925 |
SDValue Chain = CurDAG->getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, |
| 4926 |
ChainInput, N->getOperand(0)); |
4926 |
ChainInput, N->getOperand(0)); |
| 4927 |
|
4927 |
|
| 4928 |
CurDAG->SelectNodeTo(N, Opcode, MVT::Other, SDValue(NewDecrement, 0), |
4928 |
CurDAG->SelectNodeTo(N, Opcode, MVT::Other, SDValue(NewDecrement, 0), |
| 4929 |
N->getOperand(4), Chain); |
4929 |
N->getOperand(4), Chain); |
| 4930 |
return true; |
4930 |
return true; |
| 4931 |
} |
4931 |
} |
| 4932 |
|
4932 |
|
| 4933 |
bool PPCDAGToDAGISel::tryAsSingleRLWINM(SDNode *N) { |
4933 |
bool PPCDAGToDAGISel::tryAsSingleRLWINM(SDNode *N) { |
| 4934 |
assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); |
4934 |
assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); |
| 4935 |
unsigned Imm; |
4935 |
unsigned Imm; |
| 4936 |
if (!isInt32Immediate(N->getOperand(1), Imm)) |
4936 |
if (!isInt32Immediate(N->getOperand(1), Imm)) |
| 4937 |
return false; |
4937 |
return false; |
| 4938 |
|
4938 |
|
| 4939 |
SDLoc dl(N); |
4939 |
SDLoc dl(N); |
| 4940 |
SDValue Val = N->getOperand(0); |
4940 |
SDValue Val = N->getOperand(0); |
| 4941 |
unsigned SH, MB, ME; |
4941 |
unsigned SH, MB, ME; |
| 4942 |
// If this is an and of a value rotated between 0 and 31 bits and then and'd |
4942 |
// If this is an and of a value rotated between 0 and 31 bits and then and'd |
| 4943 |
// with a mask, emit rlwinm |
4943 |
// with a mask, emit rlwinm |
| 4944 |
if (isRotateAndMask(Val.getNode(), Imm, false, SH, MB, ME)) { |
4944 |
if (isRotateAndMask(Val.getNode(), Imm, false, SH, MB, ME)) { |
| 4945 |
Val = Val.getOperand(0); |
4945 |
Val = Val.getOperand(0); |
| 4946 |
SDValue Ops[] = {Val, getI32Imm(SH, dl), getI32Imm(MB, dl), |
4946 |
SDValue Ops[] = {Val, getI32Imm(SH, dl), getI32Imm(MB, dl), |
| 4947 |
getI32Imm(ME, dl)}; |
4947 |
getI32Imm(ME, dl)}; |
| 4948 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
4948 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
| 4949 |
return true; |
4949 |
return true; |
| 4950 |
} |
4950 |
} |
| 4951 |
|
4951 |
|
| 4952 |
// If this is just a masked value where the input is not handled, and |
4952 |
// If this is just a masked value where the input is not handled, and |
| 4953 |
// is not a rotate-left (handled by a pattern in the .td file), emit rlwinm |
4953 |
// is not a rotate-left (handled by a pattern in the .td file), emit rlwinm |
| 4954 |
if (isRunOfOnes(Imm, MB, ME) && Val.getOpcode() != ISD::ROTL) { |
4954 |
if (isRunOfOnes(Imm, MB, ME) && Val.getOpcode() != ISD::ROTL) { |
| 4955 |
SDValue Ops[] = {Val, getI32Imm(0, dl), getI32Imm(MB, dl), |
4955 |
SDValue Ops[] = {Val, getI32Imm(0, dl), getI32Imm(MB, dl), |
| 4956 |
getI32Imm(ME, dl)}; |
4956 |
getI32Imm(ME, dl)}; |
| 4957 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
4957 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
| 4958 |
return true; |
4958 |
return true; |
| 4959 |
} |
4959 |
} |
| 4960 |
|
4960 |
|
| 4961 |
// AND X, 0 -> 0, not "rlwinm 32". |
4961 |
// AND X, 0 -> 0, not "rlwinm 32". |
| 4962 |
if (Imm == 0) { |
4962 |
if (Imm == 0) { |
| 4963 |
ReplaceUses(SDValue(N, 0), N->getOperand(1)); |
4963 |
ReplaceUses(SDValue(N, 0), N->getOperand(1)); |
| 4964 |
return true; |
4964 |
return true; |
| 4965 |
} |
4965 |
} |
| 4966 |
|
4966 |
|
| 4967 |
return false; |
4967 |
return false; |
| 4968 |
} |
4968 |
} |
| 4969 |
|
4969 |
|
| 4970 |
bool PPCDAGToDAGISel::tryAsSingleRLWINM8(SDNode *N) { |
4970 |
bool PPCDAGToDAGISel::tryAsSingleRLWINM8(SDNode *N) { |
| 4971 |
assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); |
4971 |
assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); |
| 4972 |
uint64_t Imm64; |
4972 |
uint64_t Imm64; |
| 4973 |
if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64)) |
4973 |
if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64)) |
| 4974 |
return false; |
4974 |
return false; |
| 4975 |
|
4975 |
|
| 4976 |
unsigned MB, ME; |
4976 |
unsigned MB, ME; |
| 4977 |
if (isRunOfOnes64(Imm64, MB, ME) && MB >= 32 && MB <= ME) { |
4977 |
if (isRunOfOnes64(Imm64, MB, ME) && MB >= 32 && MB <= ME) { |
| 4978 |
// MB ME |
4978 |
// MB ME |
| 4979 |
// +----------------------+ |
4979 |
// +----------------------+ |
| 4980 |
// |xxxxxxxxxxx00011111000| |
4980 |
// |xxxxxxxxxxx00011111000| |
| 4981 |
// +----------------------+ |
4981 |
// +----------------------+ |
| 4982 |
// 0 32 64 |
4982 |
// 0 32 64 |
| 4983 |
// We can only do it if the MB is larger than 32 and MB <= ME |
4983 |
// We can only do it if the MB is larger than 32 and MB <= ME |
| 4984 |
// as RLWINM will replace the contents of [0 - 32) with [32 - 64) even |
4984 |
// as RLWINM will replace the contents of [0 - 32) with [32 - 64) even |
| 4985 |
// we didn't rotate it. |
4985 |
// we didn't rotate it. |
| 4986 |
SDLoc dl(N); |
4986 |
SDLoc dl(N); |
| 4987 |
SDValue Ops[] = {N->getOperand(0), getI64Imm(0, dl), getI64Imm(MB - 32, dl), |
4987 |
SDValue Ops[] = {N->getOperand(0), getI64Imm(0, dl), getI64Imm(MB - 32, dl), |
| 4988 |
getI64Imm(ME - 32, dl)}; |
4988 |
getI64Imm(ME - 32, dl)}; |
| 4989 |
CurDAG->SelectNodeTo(N, PPC::RLWINM8, MVT::i64, Ops); |
4989 |
CurDAG->SelectNodeTo(N, PPC::RLWINM8, MVT::i64, Ops); |
| 4990 |
return true; |
4990 |
return true; |
| 4991 |
} |
4991 |
} |
| 4992 |
|
4992 |
|
| 4993 |
return false; |
4993 |
return false; |
| 4994 |
} |
4994 |
} |
| 4995 |
|
4995 |
|
| 4996 |
bool PPCDAGToDAGISel::tryAsPairOfRLDICL(SDNode *N) { |
4996 |
bool PPCDAGToDAGISel::tryAsPairOfRLDICL(SDNode *N) { |
| 4997 |
assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); |
4997 |
assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); |
| 4998 |
uint64_t Imm64; |
4998 |
uint64_t Imm64; |
| 4999 |
if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64)) |
4999 |
if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64)) |
| 5000 |
return false; |
5000 |
return false; |
| 5001 |
|
5001 |
|
| 5002 |
// Do nothing if it is 16-bit imm as the pattern in the .td file handle |
5002 |
// Do nothing if it is 16-bit imm as the pattern in the .td file handle |
| 5003 |
// it well with "andi.". |
5003 |
// it well with "andi.". |
| 5004 |
if (isUInt<16>(Imm64)) |
5004 |
if (isUInt<16>(Imm64)) |
| 5005 |
return false; |
5005 |
return false; |
| 5006 |
|
5006 |
|
| 5007 |
SDLoc Loc(N); |
5007 |
SDLoc Loc(N); |
| 5008 |
SDValue Val = N->getOperand(0); |
5008 |
SDValue Val = N->getOperand(0); |
| 5009 |
|
5009 |
|
| 5010 |
// Optimized with two rldicl's as follows: |
5010 |
// Optimized with two rldicl's as follows: |
| 5011 |
// Add missing bits on left to the mask and check that the mask is a |
5011 |
// Add missing bits on left to the mask and check that the mask is a |
| 5012 |
// wrapped run of ones, i.e. |
5012 |
// wrapped run of ones, i.e. |
| 5013 |
// Change pattern |0001111100000011111111| |
5013 |
// Change pattern |0001111100000011111111| |
| 5014 |
// to |1111111100000011111111|. |
5014 |
// to |1111111100000011111111|. |
| 5015 |
unsigned NumOfLeadingZeros = llvm::countl_zero(Imm64); |
5015 |
unsigned NumOfLeadingZeros = llvm::countl_zero(Imm64); |
| 5016 |
if (NumOfLeadingZeros != 0) |
5016 |
if (NumOfLeadingZeros != 0) |
| 5017 |
Imm64 |= maskLeadingOnes(NumOfLeadingZeros); |
5017 |
Imm64 |= maskLeadingOnes(NumOfLeadingZeros); |
| 5018 |
|
5018 |
|
| 5019 |
unsigned MB, ME; |
5019 |
unsigned MB, ME; |
| 5020 |
if (!isRunOfOnes64(Imm64, MB, ME)) |
5020 |
if (!isRunOfOnes64(Imm64, MB, ME)) |
| 5021 |
return false; |
5021 |
return false; |
| 5022 |
|
5022 |
|
| 5023 |
// ME MB MB-ME+63 |
5023 |
// ME MB MB-ME+63 |
| 5024 |
// +----------------------+ +----------------------+ |
5024 |
// +----------------------+ +----------------------+ |
| 5025 |
// |1111111100000011111111| -> |0000001111111111111111| |
5025 |
// |1111111100000011111111| -> |0000001111111111111111| |
| 5026 |
// +----------------------+ +----------------------+ |
5026 |
// +----------------------+ +----------------------+ |
| 5027 |
// 0 63 0 63 |
5027 |
// 0 63 0 63 |
| 5028 |
// There are ME + 1 ones on the left and (MB - ME + 63) & 63 zeros in between. |
5028 |
// There are ME + 1 ones on the left and (MB - ME + 63) & 63 zeros in between. |
| 5029 |
unsigned OnesOnLeft = ME + 1; |
5029 |
unsigned OnesOnLeft = ME + 1; |
| 5030 |
unsigned ZerosInBetween = (MB - ME + 63) & 63; |
5030 |
unsigned ZerosInBetween = (MB - ME + 63) & 63; |
| 5031 |
// Rotate left by OnesOnLeft (so leading ones are now trailing ones) and clear |
5031 |
// Rotate left by OnesOnLeft (so leading ones are now trailing ones) and clear |
| 5032 |
// on the left the bits that are already zeros in the mask. |
5032 |
// on the left the bits that are already zeros in the mask. |
| 5033 |
Val = SDValue(CurDAG->getMachineNode(PPC::RLDICL, Loc, MVT::i64, Val, |
5033 |
Val = SDValue(CurDAG->getMachineNode(PPC::RLDICL, Loc, MVT::i64, Val, |
| 5034 |
getI64Imm(OnesOnLeft, Loc), |
5034 |
getI64Imm(OnesOnLeft, Loc), |
| 5035 |
getI64Imm(ZerosInBetween, Loc)), |
5035 |
getI64Imm(ZerosInBetween, Loc)), |
| 5036 |
0); |
5036 |
0); |
| 5037 |
// MB-ME+63 ME MB |
5037 |
// MB-ME+63 ME MB |
| 5038 |
// +----------------------+ +----------------------+ |
5038 |
// +----------------------+ +----------------------+ |
| 5039 |
// |0000001111111111111111| -> |0001111100000011111111| |
5039 |
// |0000001111111111111111| -> |0001111100000011111111| |
| 5040 |
// +----------------------+ +----------------------+ |
5040 |
// +----------------------+ +----------------------+ |
| 5041 |
// 0 63 0 63 |
5041 |
// 0 63 0 63 |
| 5042 |
// Rotate back by 64 - OnesOnLeft to undo previous rotate. Then clear on the |
5042 |
// Rotate back by 64 - OnesOnLeft to undo previous rotate. Then clear on the |
| 5043 |
// left the number of ones we previously added. |
5043 |
// left the number of ones we previously added. |
| 5044 |
SDValue Ops[] = {Val, getI64Imm(64 - OnesOnLeft, Loc), |
5044 |
SDValue Ops[] = {Val, getI64Imm(64 - OnesOnLeft, Loc), |
| 5045 |
getI64Imm(NumOfLeadingZeros, Loc)}; |
5045 |
getI64Imm(NumOfLeadingZeros, Loc)}; |
| 5046 |
CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops); |
5046 |
CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops); |
| 5047 |
return true; |
5047 |
return true; |
| 5048 |
} |
5048 |
} |
| 5049 |
|
5049 |
|
| 5050 |
bool PPCDAGToDAGISel::tryAsSingleRLWIMI(SDNode *N) { |
5050 |
bool PPCDAGToDAGISel::tryAsSingleRLWIMI(SDNode *N) { |
| 5051 |
assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); |
5051 |
assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); |
| 5052 |
unsigned Imm; |
5052 |
unsigned Imm; |
| 5053 |
if (!isInt32Immediate(N->getOperand(1), Imm)) |
5053 |
if (!isInt32Immediate(N->getOperand(1), Imm)) |
| 5054 |
return false; |
5054 |
return false; |
| 5055 |
|
5055 |
|
| 5056 |
SDValue Val = N->getOperand(0); |
5056 |
SDValue Val = N->getOperand(0); |
| 5057 |
unsigned Imm2; |
5057 |
unsigned Imm2; |
| 5058 |
// ISD::OR doesn't get all the bitfield insertion fun. |
5058 |
// ISD::OR doesn't get all the bitfield insertion fun. |
| 5059 |
// (and (or x, c1), c2) where isRunOfOnes(~(c1^c2)) might be a |
5059 |
// (and (or x, c1), c2) where isRunOfOnes(~(c1^c2)) might be a |
| 5060 |
// bitfield insert. |
5060 |
// bitfield insert. |
| 5061 |
if (Val.getOpcode() != ISD::OR || !isInt32Immediate(Val.getOperand(1), Imm2)) |
5061 |
if (Val.getOpcode() != ISD::OR || !isInt32Immediate(Val.getOperand(1), Imm2)) |
| 5062 |
return false; |
5062 |
return false; |
| 5063 |
|
5063 |
|
| 5064 |
// The idea here is to check whether this is equivalent to: |
5064 |
// The idea here is to check whether this is equivalent to: |
| 5065 |
// (c1 & m) | (x & ~m) |
5065 |
// (c1 & m) | (x & ~m) |
| 5066 |
// where m is a run-of-ones mask. The logic here is that, for each bit in |
5066 |
// where m is a run-of-ones mask. The logic here is that, for each bit in |
| 5067 |
// c1 and c2: |
5067 |
// c1 and c2: |
| 5068 |
// - if both are 1, then the output will be 1. |
5068 |
// - if both are 1, then the output will be 1. |
| 5069 |
// - if both are 0, then the output will be 0. |
5069 |
// - if both are 0, then the output will be 0. |
| 5070 |
// - if the bit in c1 is 0, and the bit in c2 is 1, then the output will |
5070 |
// - if the bit in c1 is 0, and the bit in c2 is 1, then the output will |
| 5071 |
// come from x. |
5071 |
// come from x. |
| 5072 |
// - if the bit in c1 is 1, and the bit in c2 is 0, then the output will |
5072 |
// - if the bit in c1 is 1, and the bit in c2 is 0, then the output will |
| 5073 |
// be 0. |
5073 |
// be 0. |
| 5074 |
// If that last condition is never the case, then we can form m from the |
5074 |
// If that last condition is never the case, then we can form m from the |
| 5075 |
// bits that are the same between c1 and c2. |
5075 |
// bits that are the same between c1 and c2. |
| 5076 |
unsigned MB, ME; |
5076 |
unsigned MB, ME; |
| 5077 |
if (isRunOfOnes(~(Imm ^ Imm2), MB, ME) && !(~Imm & Imm2)) { |
5077 |
if (isRunOfOnes(~(Imm ^ Imm2), MB, ME) && !(~Imm & Imm2)) { |
| 5078 |
SDLoc dl(N); |
5078 |
SDLoc dl(N); |
| 5079 |
SDValue Ops[] = {Val.getOperand(0), Val.getOperand(1), getI32Imm(0, dl), |
5079 |
SDValue Ops[] = {Val.getOperand(0), Val.getOperand(1), getI32Imm(0, dl), |
| 5080 |
getI32Imm(MB, dl), getI32Imm(ME, dl)}; |
5080 |
getI32Imm(MB, dl), getI32Imm(ME, dl)}; |
| 5081 |
ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops)); |
5081 |
ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops)); |
| 5082 |
return true; |
5082 |
return true; |
| 5083 |
} |
5083 |
} |
| 5084 |
|
5084 |
|
| 5085 |
return false; |
5085 |
return false; |
| 5086 |
} |
5086 |
} |
| 5087 |
|
5087 |
|
| 5088 |
bool PPCDAGToDAGISel::tryAsSingleRLDICL(SDNode *N) { |
5088 |
bool PPCDAGToDAGISel::tryAsSingleRLDICL(SDNode *N) { |
| 5089 |
assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); |
5089 |
assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); |
| 5090 |
uint64_t Imm64; |
5090 |
uint64_t Imm64; |
| 5091 |
if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || !isMask_64(Imm64)) |
5091 |
if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || !isMask_64(Imm64)) |
| 5092 |
return false; |
5092 |
return false; |
| 5093 |
|
5093 |
|
| 5094 |
// If this is a 64-bit zero-extension mask, emit rldicl. |
5094 |
// If this is a 64-bit zero-extension mask, emit rldicl. |
| 5095 |
unsigned MB = 64 - llvm::countr_one(Imm64); |
5095 |
unsigned MB = 64 - llvm::countr_one(Imm64); |
| 5096 |
unsigned SH = 0; |
5096 |
unsigned SH = 0; |
| 5097 |
unsigned Imm; |
5097 |
unsigned Imm; |
| 5098 |
SDValue Val = N->getOperand(0); |
5098 |
SDValue Val = N->getOperand(0); |
| 5099 |
SDLoc dl(N); |
5099 |
SDLoc dl(N); |
| 5100 |
|
5100 |
|
| 5101 |
if (Val.getOpcode() == ISD::ANY_EXTEND) { |
5101 |
if (Val.getOpcode() == ISD::ANY_EXTEND) { |
| 5102 |
auto Op0 = Val.getOperand(0); |
5102 |
auto Op0 = Val.getOperand(0); |
| 5103 |
if (Op0.getOpcode() == ISD::SRL && |
5103 |
if (Op0.getOpcode() == ISD::SRL && |
| 5104 |
isInt32Immediate(Op0.getOperand(1).getNode(), Imm) && Imm <= MB) { |
5104 |
isInt32Immediate(Op0.getOperand(1).getNode(), Imm) && Imm <= MB) { |
| 5105 |
|
5105 |
|
| 5106 |
auto ResultType = Val.getNode()->getValueType(0); |
5106 |
auto ResultType = Val.getNode()->getValueType(0); |
| 5107 |
auto ImDef = CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, ResultType); |
5107 |
auto ImDef = CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, ResultType); |
| 5108 |
SDValue IDVal(ImDef, 0); |
5108 |
SDValue IDVal(ImDef, 0); |
| 5109 |
|
5109 |
|
| 5110 |
Val = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, ResultType, |
5110 |
Val = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, ResultType, |
| 5111 |
IDVal, Op0.getOperand(0), |
5111 |
IDVal, Op0.getOperand(0), |
| 5112 |
getI32Imm(1, dl)), |
5112 |
getI32Imm(1, dl)), |
| 5113 |
0); |
5113 |
0); |
| 5114 |
SH = 64 - Imm; |
5114 |
SH = 64 - Imm; |
| 5115 |
} |
5115 |
} |
| 5116 |
} |
5116 |
} |
| 5117 |
|
5117 |
|
| 5118 |
// If the operand is a logical right shift, we can fold it into this |
5118 |
// If the operand is a logical right shift, we can fold it into this |
| 5119 |
// instruction: rldicl(rldicl(x, 64-n, n), 0, mb) -> rldicl(x, 64-n, mb) |
5119 |
// instruction: rldicl(rldicl(x, 64-n, n), 0, mb) -> rldicl(x, 64-n, mb) |
| 5120 |
// for n <= mb. The right shift is really a left rotate followed by a |
5120 |
// for n <= mb. The right shift is really a left rotate followed by a |
| 5121 |
// mask, and this mask is a more-restrictive sub-mask of the mask implied |
5121 |
// mask, and this mask is a more-restrictive sub-mask of the mask implied |
| 5122 |
// by the shift. |
5122 |
// by the shift. |
| 5123 |
if (Val.getOpcode() == ISD::SRL && |
5123 |
if (Val.getOpcode() == ISD::SRL && |
| 5124 |
isInt32Immediate(Val.getOperand(1).getNode(), Imm) && Imm <= MB) { |
5124 |
isInt32Immediate(Val.getOperand(1).getNode(), Imm) && Imm <= MB) { |
| 5125 |
assert(Imm < 64 && "Illegal shift amount"); |
5125 |
assert(Imm < 64 && "Illegal shift amount"); |
| 5126 |
Val = Val.getOperand(0); |
5126 |
Val = Val.getOperand(0); |
| 5127 |
SH = 64 - Imm; |
5127 |
SH = 64 - Imm; |
| 5128 |
} |
5128 |
} |
| 5129 |
|
5129 |
|
| 5130 |
SDValue Ops[] = {Val, getI32Imm(SH, dl), getI32Imm(MB, dl)}; |
5130 |
SDValue Ops[] = {Val, getI32Imm(SH, dl), getI32Imm(MB, dl)}; |
| 5131 |
CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops); |
5131 |
CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops); |
| 5132 |
return true; |
5132 |
return true; |
| 5133 |
} |
5133 |
} |
| 5134 |
|
5134 |
|
| 5135 |
bool PPCDAGToDAGISel::tryAsSingleRLDICR(SDNode *N) { |
5135 |
bool PPCDAGToDAGISel::tryAsSingleRLDICR(SDNode *N) { |
| 5136 |
assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); |
5136 |
assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); |
| 5137 |
uint64_t Imm64; |
5137 |
uint64_t Imm64; |
| 5138 |
if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || |
5138 |
if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || |
| 5139 |
!isMask_64(~Imm64)) |
5139 |
!isMask_64(~Imm64)) |
| 5140 |
return false; |
5140 |
return false; |
| 5141 |
|
5141 |
|
| 5142 |
// If this is a negated 64-bit zero-extension mask, |
5142 |
// If this is a negated 64-bit zero-extension mask, |
| 5143 |
// i.e. the immediate is a sequence of ones from most significant side |
5143 |
// i.e. the immediate is a sequence of ones from most significant side |
| 5144 |
// and all zero for reminder, we should use rldicr. |
5144 |
// and all zero for reminder, we should use rldicr. |
| 5145 |
unsigned MB = 63 - llvm::countr_one(~Imm64); |
5145 |
unsigned MB = 63 - llvm::countr_one(~Imm64); |
| 5146 |
unsigned SH = 0; |
5146 |
unsigned SH = 0; |
| 5147 |
SDLoc dl(N); |
5147 |
SDLoc dl(N); |
| 5148 |
SDValue Ops[] = {N->getOperand(0), getI32Imm(SH, dl), getI32Imm(MB, dl)}; |
5148 |
SDValue Ops[] = {N->getOperand(0), getI32Imm(SH, dl), getI32Imm(MB, dl)}; |
| 5149 |
CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, Ops); |
5149 |
CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, Ops); |
| 5150 |
return true; |
5150 |
return true; |
| 5151 |
} |
5151 |
} |
| 5152 |
|
5152 |
|
| 5153 |
bool PPCDAGToDAGISel::tryAsSingleRLDIMI(SDNode *N) { |
5153 |
bool PPCDAGToDAGISel::tryAsSingleRLDIMI(SDNode *N) { |
| 5154 |
assert(N->getOpcode() == ISD::OR && "ISD::OR SDNode expected"); |
5154 |
assert(N->getOpcode() == ISD::OR && "ISD::OR SDNode expected"); |
| 5155 |
uint64_t Imm64; |
5155 |
uint64_t Imm64; |
| 5156 |
unsigned MB, ME; |
5156 |
unsigned MB, ME; |
| 5157 |
SDValue N0 = N->getOperand(0); |
5157 |
SDValue N0 = N->getOperand(0); |
| 5158 |
|
5158 |
|
| 5159 |
// We won't get fewer instructions if the imm is 32-bit integer. |
5159 |
// We won't get fewer instructions if the imm is 32-bit integer. |
| 5160 |
// rldimi requires the imm to have consecutive ones with both sides zero. |
5160 |
// rldimi requires the imm to have consecutive ones with both sides zero. |
| 5161 |
// Also, make sure the first Op has only one use, otherwise this may increase |
5161 |
// Also, make sure the first Op has only one use, otherwise this may increase |
| 5162 |
// register pressure since rldimi is destructive. |
5162 |
// register pressure since rldimi is destructive. |
| 5163 |
if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || |
5163 |
if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || |
| 5164 |
isUInt<32>(Imm64) || !isRunOfOnes64(Imm64, MB, ME) || !N0.hasOneUse()) |
5164 |
isUInt<32>(Imm64) || !isRunOfOnes64(Imm64, MB, ME) || !N0.hasOneUse()) |
| 5165 |
return false; |
5165 |
return false; |
| 5166 |
|
5166 |
|
| 5167 |
unsigned SH = 63 - ME; |
5167 |
unsigned SH = 63 - ME; |
| 5168 |
SDLoc Dl(N); |
5168 |
SDLoc Dl(N); |
| 5169 |
// Use select64Imm for making LI instr instead of directly putting Imm64 |
5169 |
// Use select64Imm for making LI instr instead of directly putting Imm64 |
| 5170 |
SDValue Ops[] = { |
5170 |
SDValue Ops[] = { |
| 5171 |
N->getOperand(0), |
5171 |
N->getOperand(0), |
| 5172 |
SDValue(selectI64Imm(CurDAG, getI64Imm(-1, Dl).getNode()), 0), |
5172 |
SDValue(selectI64Imm(CurDAG, getI64Imm(-1, Dl).getNode()), 0), |
| 5173 |
getI32Imm(SH, Dl), getI32Imm(MB, Dl)}; |
5173 |
getI32Imm(SH, Dl), getI32Imm(MB, Dl)}; |
| 5174 |
CurDAG->SelectNodeTo(N, PPC::RLDIMI, MVT::i64, Ops); |
5174 |
CurDAG->SelectNodeTo(N, PPC::RLDIMI, MVT::i64, Ops); |
| 5175 |
return true; |
5175 |
return true; |
| 5176 |
} |
5176 |
} |
| 5177 |
|
5177 |
|
| 5178 |
// Select - Convert the specified operand from a target-independent to a |
5178 |
// Select - Convert the specified operand from a target-independent to a |
| 5179 |
// target-specific node if it hasn't already been changed. |
5179 |
// target-specific node if it hasn't already been changed. |
| 5180 |
void PPCDAGToDAGISel::Select(SDNode *N) { |
5180 |
void PPCDAGToDAGISel::Select(SDNode *N) { |
| 5181 |
SDLoc dl(N); |
5181 |
SDLoc dl(N); |
| 5182 |
if (N->isMachineOpcode()) { |
5182 |
if (N->isMachineOpcode()) { |
| 5183 |
N->setNodeId(-1); |
5183 |
N->setNodeId(-1); |
| 5184 |
return; // Already selected. |
5184 |
return; // Already selected. |
| 5185 |
} |
5185 |
} |
| 5186 |
|
5186 |
|
| 5187 |
// In case any misguided DAG-level optimizations form an ADD with a |
5187 |
// In case any misguided DAG-level optimizations form an ADD with a |
| 5188 |
// TargetConstant operand, crash here instead of miscompiling (by selecting |
5188 |
// TargetConstant operand, crash here instead of miscompiling (by selecting |
| 5189 |
// an r+r add instead of some kind of r+i add). |
5189 |
// an r+r add instead of some kind of r+i add). |
| 5190 |
if (N->getOpcode() == ISD::ADD && |
5190 |
if (N->getOpcode() == ISD::ADD && |
| 5191 |
N->getOperand(1).getOpcode() == ISD::TargetConstant) |
5191 |
N->getOperand(1).getOpcode() == ISD::TargetConstant) |
| 5192 |
llvm_unreachable("Invalid ADD with TargetConstant operand"); |
5192 |
llvm_unreachable("Invalid ADD with TargetConstant operand"); |
| 5193 |
|
5193 |
|
| 5194 |
// Try matching complex bit permutations before doing anything else. |
5194 |
// Try matching complex bit permutations before doing anything else. |
| 5195 |
if (tryBitPermutation(N)) |
5195 |
if (tryBitPermutation(N)) |
| 5196 |
return; |
5196 |
return; |
| 5197 |
|
5197 |
|
| 5198 |
// Try to emit integer compares as GPR-only sequences (i.e. no use of CR). |
5198 |
// Try to emit integer compares as GPR-only sequences (i.e. no use of CR). |
| 5199 |
if (tryIntCompareInGPR(N)) |
5199 |
if (tryIntCompareInGPR(N)) |
| 5200 |
return; |
5200 |
return; |
| 5201 |
|
5201 |
|
| 5202 |
switch (N->getOpcode()) { |
5202 |
switch (N->getOpcode()) { |
| 5203 |
default: break; |
5203 |
default: break; |
| 5204 |
|
5204 |
|
| 5205 |
case ISD::Constant: |
5205 |
case ISD::Constant: |
| 5206 |
if (N->getValueType(0) == MVT::i64) { |
5206 |
if (N->getValueType(0) == MVT::i64) { |
| 5207 |
ReplaceNode(N, selectI64Imm(CurDAG, N)); |
5207 |
ReplaceNode(N, selectI64Imm(CurDAG, N)); |
| 5208 |
return; |
5208 |
return; |
| 5209 |
} |
5209 |
} |
| 5210 |
break; |
5210 |
break; |
| 5211 |
|
5211 |
|
| 5212 |
case ISD::INTRINSIC_VOID: { |
5212 |
case ISD::INTRINSIC_VOID: { |
| 5213 |
auto IntrinsicID = N->getConstantOperandVal(1); |
5213 |
auto IntrinsicID = N->getConstantOperandVal(1); |
| 5214 |
if (IntrinsicID != Intrinsic::ppc_tdw && IntrinsicID != Intrinsic::ppc_tw && |
5214 |
if (IntrinsicID != Intrinsic::ppc_tdw && IntrinsicID != Intrinsic::ppc_tw && |
| 5215 |
IntrinsicID != Intrinsic::ppc_trapd && |
5215 |
IntrinsicID != Intrinsic::ppc_trapd && |
| 5216 |
IntrinsicID != Intrinsic::ppc_trap) |
5216 |
IntrinsicID != Intrinsic::ppc_trap) |
| 5217 |
break; |
5217 |
break; |
| 5218 |
unsigned Opcode = (IntrinsicID == Intrinsic::ppc_tdw || |
5218 |
unsigned Opcode = (IntrinsicID == Intrinsic::ppc_tdw || |
| 5219 |
IntrinsicID == Intrinsic::ppc_trapd) |
5219 |
IntrinsicID == Intrinsic::ppc_trapd) |
| 5220 |
? PPC::TDI |
5220 |
? PPC::TDI |
| 5221 |
: PPC::TWI; |
5221 |
: PPC::TWI; |
| 5222 |
SmallVector OpsWithMD; |
5222 |
SmallVector OpsWithMD; |
| 5223 |
unsigned MDIndex; |
5223 |
unsigned MDIndex; |
| 5224 |
if (IntrinsicID == Intrinsic::ppc_tdw || |
5224 |
if (IntrinsicID == Intrinsic::ppc_tdw || |
| 5225 |
IntrinsicID == Intrinsic::ppc_tw) { |
5225 |
IntrinsicID == Intrinsic::ppc_tw) { |
| 5226 |
SDValue Ops[] = {N->getOperand(4), N->getOperand(2), N->getOperand(3)}; |
5226 |
SDValue Ops[] = {N->getOperand(4), N->getOperand(2), N->getOperand(3)}; |
| 5227 |
int16_t SImmOperand2; |
5227 |
int16_t SImmOperand2; |
| 5228 |
int16_t SImmOperand3; |
5228 |
int16_t SImmOperand3; |
| 5229 |
int16_t SImmOperand4; |
5229 |
int16_t SImmOperand4; |
| 5230 |
bool isOperand2IntS16Immediate = |
5230 |
bool isOperand2IntS16Immediate = |
| 5231 |
isIntS16Immediate(N->getOperand(2), SImmOperand2); |
5231 |
isIntS16Immediate(N->getOperand(2), SImmOperand2); |
| 5232 |
bool isOperand3IntS16Immediate = |
5232 |
bool isOperand3IntS16Immediate = |
| 5233 |
isIntS16Immediate(N->getOperand(3), SImmOperand3); |
5233 |
isIntS16Immediate(N->getOperand(3), SImmOperand3); |
| 5234 |
// We will emit PPC::TD or PPC::TW if the 2nd and 3rd operands are reg + |
5234 |
// We will emit PPC::TD or PPC::TW if the 2nd and 3rd operands are reg + |
| 5235 |
// reg or imm + imm. The imm + imm form will be optimized to either an |
5235 |
// reg or imm + imm. The imm + imm form will be optimized to either an |
| 5236 |
// unconditional trap or a nop in a later pass. |
5236 |
// unconditional trap or a nop in a later pass. |
| 5237 |
if (isOperand2IntS16Immediate == isOperand3IntS16Immediate) |
5237 |
if (isOperand2IntS16Immediate == isOperand3IntS16Immediate) |
| 5238 |
Opcode = IntrinsicID == Intrinsic::ppc_tdw ? PPC::TD : PPC::TW; |
5238 |
Opcode = IntrinsicID == Intrinsic::ppc_tdw ? PPC::TD : PPC::TW; |
| 5239 |
else if (isOperand3IntS16Immediate) |
5239 |
else if (isOperand3IntS16Immediate) |
| 5240 |
// The 2nd and 3rd operands are reg + imm. |
5240 |
// The 2nd and 3rd operands are reg + imm. |
| 5241 |
Ops[2] = getI32Imm(int(SImmOperand3) & 0xFFFF, dl); |
5241 |
Ops[2] = getI32Imm(int(SImmOperand3) & 0xFFFF, dl); |
| 5242 |
else { |
5242 |
else { |
| 5243 |
// The 2nd and 3rd operands are imm + reg. |
5243 |
// The 2nd and 3rd operands are imm + reg. |
| 5244 |
bool isOperand4IntS16Immediate = |
5244 |
bool isOperand4IntS16Immediate = |
| 5245 |
isIntS16Immediate(N->getOperand(4), SImmOperand4); |
5245 |
isIntS16Immediate(N->getOperand(4), SImmOperand4); |
| 5246 |
(void)isOperand4IntS16Immediate; |
5246 |
(void)isOperand4IntS16Immediate; |
| 5247 |
assert(isOperand4IntS16Immediate && |
5247 |
assert(isOperand4IntS16Immediate && |
| 5248 |
"The 4th operand is not an Immediate"); |
5248 |
"The 4th operand is not an Immediate"); |
| 5249 |
// We need to flip the condition immediate TO. |
5249 |
// We need to flip the condition immediate TO. |
| 5250 |
int16_t TO = int(SImmOperand4) & 0x1F; |
5250 |
int16_t TO = int(SImmOperand4) & 0x1F; |
| 5251 |
// We swap the first and second bit of TO if they are not same. |
5251 |
// We swap the first and second bit of TO if they are not same. |
| 5252 |
if ((TO & 0x1) != ((TO & 0x2) >> 1)) |
5252 |
if ((TO & 0x1) != ((TO & 0x2) >> 1)) |
| 5253 |
TO = (TO & 0x1) ? TO + 1 : TO - 1; |
5253 |
TO = (TO & 0x1) ? TO + 1 : TO - 1; |
| 5254 |
// We swap the fourth and fifth bit of TO if they are not same. |
5254 |
// We swap the fourth and fifth bit of TO if they are not same. |
| 5255 |
if ((TO & 0x8) != ((TO & 0x10) >> 1)) |
5255 |
if ((TO & 0x8) != ((TO & 0x10) >> 1)) |
| 5256 |
TO = (TO & 0x8) ? TO + 8 : TO - 8; |
5256 |
TO = (TO & 0x8) ? TO + 8 : TO - 8; |
| 5257 |
Ops[0] = getI32Imm(TO, dl); |
5257 |
Ops[0] = getI32Imm(TO, dl); |
| 5258 |
Ops[1] = N->getOperand(3); |
5258 |
Ops[1] = N->getOperand(3); |
| 5259 |
Ops[2] = getI32Imm(int(SImmOperand2) & 0xFFFF, dl); |
5259 |
Ops[2] = getI32Imm(int(SImmOperand2) & 0xFFFF, dl); |
| 5260 |
} |
5260 |
} |
| 5261 |
OpsWithMD = {Ops[0], Ops[1], Ops[2]}; |
5261 |
OpsWithMD = {Ops[0], Ops[1], Ops[2]}; |
| 5262 |
MDIndex = 5; |
5262 |
MDIndex = 5; |
| 5263 |
} else { |
5263 |
} else { |
| 5264 |
OpsWithMD = {getI32Imm(24, dl), N->getOperand(2), getI32Imm(0, dl)}; |
5264 |
OpsWithMD = {getI32Imm(24, dl), N->getOperand(2), getI32Imm(0, dl)}; |
| 5265 |
MDIndex = 3; |
5265 |
MDIndex = 3; |
| 5266 |
} |
5266 |
} |
| 5267 |
|
5267 |
|
| 5268 |
if (N->getNumOperands() > MDIndex) { |
5268 |
if (N->getNumOperands() > MDIndex) { |
| 5269 |
SDValue MDV = N->getOperand(MDIndex); |
5269 |
SDValue MDV = N->getOperand(MDIndex); |
| 5270 |
const MDNode *MD = cast(MDV)->getMD(); |
5270 |
const MDNode *MD = cast(MDV)->getMD(); |
| 5271 |
assert(MD->getNumOperands() != 0 && "Empty MDNode in operands!"); |
5271 |
assert(MD->getNumOperands() != 0 && "Empty MDNode in operands!"); |
| 5272 |
assert((isa(MD->getOperand(0)) && cast( |
5272 |
assert((isa(MD->getOperand(0)) && cast( |
| 5273 |
MD->getOperand(0))->getString().equals("ppc-trap-reason")) |
5273 |
MD->getOperand(0))->getString().equals("ppc-trap-reason")) |
| 5274 |
&& "Unsupported annotation data type!"); |
5274 |
&& "Unsupported annotation data type!"); |
| 5275 |
for (unsigned i = 1; i < MD->getNumOperands(); i++) { |
5275 |
for (unsigned i = 1; i < MD->getNumOperands(); i++) { |
| 5276 |
assert(isa(MD->getOperand(i)) && |
5276 |
assert(isa(MD->getOperand(i)) && |
| 5277 |
"Invalid data type for annotation ppc-trap-reason!"); |
5277 |
"Invalid data type for annotation ppc-trap-reason!"); |
| 5278 |
OpsWithMD.push_back( |
5278 |
OpsWithMD.push_back( |
| 5279 |
getI32Imm(std::stoi(cast( |
5279 |
getI32Imm(std::stoi(cast( |
| 5280 |
MD->getOperand(i))->getString().str()), dl)); |
5280 |
MD->getOperand(i))->getString().str()), dl)); |
| 5281 |
} |
5281 |
} |
| 5282 |
} |
5282 |
} |
| 5283 |
OpsWithMD.push_back(N->getOperand(0)); // chain |
5283 |
OpsWithMD.push_back(N->getOperand(0)); // chain |
| 5284 |
CurDAG->SelectNodeTo(N, Opcode, MVT::Other, OpsWithMD); |
5284 |
CurDAG->SelectNodeTo(N, Opcode, MVT::Other, OpsWithMD); |
| 5285 |
return; |
5285 |
return; |
| 5286 |
} |
5286 |
} |
| 5287 |
|
5287 |
|
| 5288 |
case ISD::INTRINSIC_WO_CHAIN: { |
5288 |
case ISD::INTRINSIC_WO_CHAIN: { |
| 5289 |
// We emit the PPC::FSELS instruction here because of type conflicts with |
5289 |
// We emit the PPC::FSELS instruction here because of type conflicts with |
| 5290 |
// the comparison operand. The FSELS instruction is defined to use an 8-byte |
5290 |
// the comparison operand. The FSELS instruction is defined to use an 8-byte |
| 5291 |
// comparison like the FSELD version. The fsels intrinsic takes a 4-byte |
5291 |
// comparison like the FSELD version. The fsels intrinsic takes a 4-byte |
| 5292 |
// value for the comparison. When selecting through a .td file, a type |
5292 |
// value for the comparison. When selecting through a .td file, a type |
| 5293 |
// error is raised. Must check this first so we never break on the |
5293 |
// error is raised. Must check this first so we never break on the |
| 5294 |
// !Subtarget->isISA3_1() check. |
5294 |
// !Subtarget->isISA3_1() check. |
| 5295 |
auto IntID = N->getConstantOperandVal(0); |
5295 |
auto IntID = N->getConstantOperandVal(0); |
| 5296 |
if (IntID == Intrinsic::ppc_fsels) { |
5296 |
if (IntID == Intrinsic::ppc_fsels) { |
| 5297 |
SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3)}; |
5297 |
SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3)}; |
| 5298 |
CurDAG->SelectNodeTo(N, PPC::FSELS, MVT::f32, Ops); |
5298 |
CurDAG->SelectNodeTo(N, PPC::FSELS, MVT::f32, Ops); |
| 5299 |
return; |
5299 |
return; |
| 5300 |
} |
5300 |
} |
| 5301 |
|
5301 |
|
| 5302 |
if (IntID == Intrinsic::ppc_bcdadd_p || IntID == Intrinsic::ppc_bcdsub_p) { |
5302 |
if (IntID == Intrinsic::ppc_bcdadd_p || IntID == Intrinsic::ppc_bcdsub_p) { |
| 5303 |
auto Pred = N->getConstantOperandVal(1); |
5303 |
auto Pred = N->getConstantOperandVal(1); |
| 5304 |
unsigned Opcode = |
5304 |
unsigned Opcode = |
| 5305 |
IntID == Intrinsic::ppc_bcdadd_p ? PPC::BCDADD_rec : PPC::BCDSUB_rec; |
5305 |
IntID == Intrinsic::ppc_bcdadd_p ? PPC::BCDADD_rec : PPC::BCDSUB_rec; |
| 5306 |
unsigned SubReg = 0; |
5306 |
unsigned SubReg = 0; |
| 5307 |
unsigned ShiftVal = 0; |
5307 |
unsigned ShiftVal = 0; |
| 5308 |
bool Reverse = false; |
5308 |
bool Reverse = false; |
| 5309 |
switch (Pred) { |
5309 |
switch (Pred) { |
| 5310 |
case 0: |
5310 |
case 0: |
| 5311 |
SubReg = PPC::sub_eq; |
5311 |
SubReg = PPC::sub_eq; |
| 5312 |
ShiftVal = 1; |
5312 |
ShiftVal = 1; |
| 5313 |
break; |
5313 |
break; |
| 5314 |
case 1: |
5314 |
case 1: |
| 5315 |
SubReg = PPC::sub_eq; |
5315 |
SubReg = PPC::sub_eq; |
| 5316 |
ShiftVal = 1; |
5316 |
ShiftVal = 1; |
| 5317 |
Reverse = true; |
5317 |
Reverse = true; |
| 5318 |
break; |
5318 |
break; |
| 5319 |
case 2: |
5319 |
case 2: |
| 5320 |
SubReg = PPC::sub_lt; |
5320 |
SubReg = PPC::sub_lt; |
| 5321 |
ShiftVal = 3; |
5321 |
ShiftVal = 3; |
| 5322 |
break; |
5322 |
break; |
| 5323 |
case 3: |
5323 |
case 3: |
| 5324 |
SubReg = PPC::sub_lt; |
5324 |
SubReg = PPC::sub_lt; |
| 5325 |
ShiftVal = 3; |
5325 |
ShiftVal = 3; |
| 5326 |
Reverse = true; |
5326 |
Reverse = true; |
| 5327 |
break; |
5327 |
break; |
| 5328 |
case 4: |
5328 |
case 4: |
| 5329 |
SubReg = PPC::sub_gt; |
5329 |
SubReg = PPC::sub_gt; |
| 5330 |
ShiftVal = 2; |
5330 |
ShiftVal = 2; |
| 5331 |
break; |
5331 |
break; |
| 5332 |
case 5: |
5332 |
case 5: |
| 5333 |
SubReg = PPC::sub_gt; |
5333 |
SubReg = PPC::sub_gt; |
| 5334 |
ShiftVal = 2; |
5334 |
ShiftVal = 2; |
| 5335 |
Reverse = true; |
5335 |
Reverse = true; |
| 5336 |
break; |
5336 |
break; |
| 5337 |
case 6: |
5337 |
case 6: |
| 5338 |
SubReg = PPC::sub_un; |
5338 |
SubReg = PPC::sub_un; |
| 5339 |
break; |
5339 |
break; |
| 5340 |
case 7: |
5340 |
case 7: |
| 5341 |
SubReg = PPC::sub_un; |
5341 |
SubReg = PPC::sub_un; |
| 5342 |
Reverse = true; |
5342 |
Reverse = true; |
| 5343 |
break; |
5343 |
break; |
| 5344 |
} |
5344 |
} |
| 5345 |
|
5345 |
|
| 5346 |
EVT VTs[] = {MVT::v16i8, MVT::Glue}; |
5346 |
EVT VTs[] = {MVT::v16i8, MVT::Glue}; |
| 5347 |
SDValue Ops[] = {N->getOperand(2), N->getOperand(3), |
5347 |
SDValue Ops[] = {N->getOperand(2), N->getOperand(3), |
| 5348 |
CurDAG->getTargetConstant(0, dl, MVT::i32)}; |
5348 |
CurDAG->getTargetConstant(0, dl, MVT::i32)}; |
| 5349 |
SDValue BCDOp = SDValue(CurDAG->getMachineNode(Opcode, dl, VTs, Ops), 0); |
5349 |
SDValue BCDOp = SDValue(CurDAG->getMachineNode(Opcode, dl, VTs, Ops), 0); |
| 5350 |
SDValue CR6Reg = CurDAG->getRegister(PPC::CR6, MVT::i32); |
5350 |
SDValue CR6Reg = CurDAG->getRegister(PPC::CR6, MVT::i32); |
| 5351 |
// On Power10, we can use SETBC[R]. On prior architectures, we have to use |
5351 |
// On Power10, we can use SETBC[R]. On prior architectures, we have to use |
| 5352 |
// MFOCRF and shift/negate the value. |
5352 |
// MFOCRF and shift/negate the value. |
| 5353 |
if (Subtarget->isISA3_1()) { |
5353 |
if (Subtarget->isISA3_1()) { |
| 5354 |
SDValue SubRegIdx = CurDAG->getTargetConstant(SubReg, dl, MVT::i32); |
5354 |
SDValue SubRegIdx = CurDAG->getTargetConstant(SubReg, dl, MVT::i32); |
| 5355 |
SDValue CRBit = SDValue( |
5355 |
SDValue CRBit = SDValue( |
| 5356 |
CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i1, |
5356 |
CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i1, |
| 5357 |
CR6Reg, SubRegIdx, BCDOp.getValue(1)), |
5357 |
CR6Reg, SubRegIdx, BCDOp.getValue(1)), |
| 5358 |
0); |
5358 |
0); |
| 5359 |
CurDAG->SelectNodeTo(N, Reverse ? PPC::SETBCR : PPC::SETBC, MVT::i32, |
5359 |
CurDAG->SelectNodeTo(N, Reverse ? PPC::SETBCR : PPC::SETBC, MVT::i32, |
| 5360 |
CRBit); |
5360 |
CRBit); |
| 5361 |
} else { |
5361 |
} else { |
| 5362 |
SDValue Move = |
5362 |
SDValue Move = |
| 5363 |
SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR6Reg, |
5363 |
SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR6Reg, |
| 5364 |
BCDOp.getValue(1)), |
5364 |
BCDOp.getValue(1)), |
| 5365 |
0); |
5365 |
0); |
| 5366 |
SDValue Ops[] = {Move, getI32Imm((32 - (4 + ShiftVal)) & 31, dl), |
5366 |
SDValue Ops[] = {Move, getI32Imm((32 - (4 + ShiftVal)) & 31, dl), |
| 5367 |
getI32Imm(31, dl), getI32Imm(31, dl)}; |
5367 |
getI32Imm(31, dl), getI32Imm(31, dl)}; |
| 5368 |
if (!Reverse) |
5368 |
if (!Reverse) |
| 5369 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
5369 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
| 5370 |
else { |
5370 |
else { |
| 5371 |
SDValue Shift = SDValue( |
5371 |
SDValue Shift = SDValue( |
| 5372 |
CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); |
5372 |
CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); |
| 5373 |
CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Shift, getI32Imm(1, dl)); |
5373 |
CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Shift, getI32Imm(1, dl)); |
| 5374 |
} |
5374 |
} |
| 5375 |
} |
5375 |
} |
| 5376 |
return; |
5376 |
return; |
| 5377 |
} |
5377 |
} |
| 5378 |
|
5378 |
|
| 5379 |
if (!Subtarget->isISA3_1()) |
5379 |
if (!Subtarget->isISA3_1()) |
| 5380 |
break; |
5380 |
break; |
| 5381 |
unsigned Opcode = 0; |
5381 |
unsigned Opcode = 0; |
| 5382 |
switch (IntID) { |
5382 |
switch (IntID) { |
| 5383 |
default: |
5383 |
default: |
| 5384 |
break; |
5384 |
break; |
| 5385 |
case Intrinsic::ppc_altivec_vstribr_p: |
5385 |
case Intrinsic::ppc_altivec_vstribr_p: |
| 5386 |
Opcode = PPC::VSTRIBR_rec; |
5386 |
Opcode = PPC::VSTRIBR_rec; |
| 5387 |
break; |
5387 |
break; |
| 5388 |
case Intrinsic::ppc_altivec_vstribl_p: |
5388 |
case Intrinsic::ppc_altivec_vstribl_p: |
| 5389 |
Opcode = PPC::VSTRIBL_rec; |
5389 |
Opcode = PPC::VSTRIBL_rec; |
| 5390 |
break; |
5390 |
break; |
| 5391 |
case Intrinsic::ppc_altivec_vstrihr_p: |
5391 |
case Intrinsic::ppc_altivec_vstrihr_p: |
| 5392 |
Opcode = PPC::VSTRIHR_rec; |
5392 |
Opcode = PPC::VSTRIHR_rec; |
| 5393 |
break; |
5393 |
break; |
| 5394 |
case Intrinsic::ppc_altivec_vstrihl_p: |
5394 |
case Intrinsic::ppc_altivec_vstrihl_p: |
| 5395 |
Opcode = PPC::VSTRIHL_rec; |
5395 |
Opcode = PPC::VSTRIHL_rec; |
| 5396 |
break; |
5396 |
break; |
| 5397 |
} |
5397 |
} |
| 5398 |
if (!Opcode) |
5398 |
if (!Opcode) |
| 5399 |
break; |
5399 |
break; |
| 5400 |
|
5400 |
|
| 5401 |
// Generate the appropriate vector string isolate intrinsic to match. |
5401 |
// Generate the appropriate vector string isolate intrinsic to match. |
| 5402 |
EVT VTs[] = {MVT::v16i8, MVT::Glue}; |
5402 |
EVT VTs[] = {MVT::v16i8, MVT::Glue}; |
| 5403 |
SDValue VecStrOp = |
5403 |
SDValue VecStrOp = |
| 5404 |
SDValue(CurDAG->getMachineNode(Opcode, dl, VTs, N->getOperand(2)), 0); |
5404 |
SDValue(CurDAG->getMachineNode(Opcode, dl, VTs, N->getOperand(2)), 0); |
| 5405 |
// Vector string isolate instructions update the EQ bit of CR6. |
5405 |
// Vector string isolate instructions update the EQ bit of CR6. |
| 5406 |
// Generate a SETBC instruction to extract the bit and place it in a GPR. |
5406 |
// Generate a SETBC instruction to extract the bit and place it in a GPR. |
| 5407 |
SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_eq, dl, MVT::i32); |
5407 |
SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_eq, dl, MVT::i32); |
| 5408 |
SDValue CR6Reg = CurDAG->getRegister(PPC::CR6, MVT::i32); |
5408 |
SDValue CR6Reg = CurDAG->getRegister(PPC::CR6, MVT::i32); |
| 5409 |
SDValue CRBit = SDValue( |
5409 |
SDValue CRBit = SDValue( |
| 5410 |
CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i1, |
5410 |
CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i1, |
| 5411 |
CR6Reg, SubRegIdx, VecStrOp.getValue(1)), |
5411 |
CR6Reg, SubRegIdx, VecStrOp.getValue(1)), |
| 5412 |
0); |
5412 |
0); |
| 5413 |
CurDAG->SelectNodeTo(N, PPC::SETBC, MVT::i32, CRBit); |
5413 |
CurDAG->SelectNodeTo(N, PPC::SETBC, MVT::i32, CRBit); |
| 5414 |
return; |
5414 |
return; |
| 5415 |
} |
5415 |
} |
| 5416 |
|
5416 |
|
| 5417 |
case ISD::SETCC: |
5417 |
case ISD::SETCC: |
| 5418 |
case ISD::STRICT_FSETCC: |
5418 |
case ISD::STRICT_FSETCC: |
| 5419 |
case ISD::STRICT_FSETCCS: |
5419 |
case ISD::STRICT_FSETCCS: |
| 5420 |
if (trySETCC(N)) |
5420 |
if (trySETCC(N)) |
| 5421 |
return; |
5421 |
return; |
| 5422 |
break; |
5422 |
break; |
| 5423 |
// These nodes will be transformed into GETtlsADDR32 node, which |
5423 |
// These nodes will be transformed into GETtlsADDR32 node, which |
| 5424 |
// later becomes BL_TLS __tls_get_addr(sym at tlsgd)@PLT |
5424 |
// later becomes BL_TLS __tls_get_addr(sym at tlsgd)@PLT |
| 5425 |
case PPCISD::ADDI_TLSLD_L_ADDR: |
5425 |
case PPCISD::ADDI_TLSLD_L_ADDR: |
| 5426 |
case PPCISD::ADDI_TLSGD_L_ADDR: { |
5426 |
case PPCISD::ADDI_TLSGD_L_ADDR: { |
| 5427 |
const Module *Mod = MF->getFunction().getParent(); |
5427 |
const Module *Mod = MF->getFunction().getParent(); |
| 5428 |
if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 || |
5428 |
if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 || |
| 5429 |
!Subtarget->isSecurePlt() || !Subtarget->isTargetELF() || |
5429 |
!Subtarget->isSecurePlt() || !Subtarget->isTargetELF() || |
| 5430 |
Mod->getPICLevel() == PICLevel::SmallPIC) |
5430 |
Mod->getPICLevel() == PICLevel::SmallPIC) |
| 5431 |
break; |
5431 |
break; |
| 5432 |
// Attach global base pointer on GETtlsADDR32 node in order to |
5432 |
// Attach global base pointer on GETtlsADDR32 node in order to |
| 5433 |
// generate secure plt code for TLS symbols. |
5433 |
// generate secure plt code for TLS symbols. |
| 5434 |
getGlobalBaseReg(); |
5434 |
getGlobalBaseReg(); |
| 5435 |
} break; |
5435 |
} break; |
| 5436 |
case PPCISD::CALL: { |
5436 |
case PPCISD::CALL: { |
| 5437 |
if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 || |
5437 |
if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 || |
| 5438 |
!TM.isPositionIndependent() || !Subtarget->isSecurePlt() || |
5438 |
!TM.isPositionIndependent() || !Subtarget->isSecurePlt() || |
| 5439 |
!Subtarget->isTargetELF()) |
5439 |
!Subtarget->isTargetELF()) |
| 5440 |
break; |
5440 |
break; |
| 5441 |
|
5441 |
|
| 5442 |
SDValue Op = N->getOperand(1); |
5442 |
SDValue Op = N->getOperand(1); |
| 5443 |
|
5443 |
|
| 5444 |
if (GlobalAddressSDNode *GA = dyn_cast(Op)) { |
5444 |
if (GlobalAddressSDNode *GA = dyn_cast(Op)) { |
| 5445 |
if (GA->getTargetFlags() == PPCII::MO_PLT) |
5445 |
if (GA->getTargetFlags() == PPCII::MO_PLT) |
| 5446 |
getGlobalBaseReg(); |
5446 |
getGlobalBaseReg(); |
| 5447 |
} |
5447 |
} |
| 5448 |
else if (ExternalSymbolSDNode *ES = dyn_cast(Op)) { |
5448 |
else if (ExternalSymbolSDNode *ES = dyn_cast(Op)) { |
| 5449 |
if (ES->getTargetFlags() == PPCII::MO_PLT) |
5449 |
if (ES->getTargetFlags() == PPCII::MO_PLT) |
| 5450 |
getGlobalBaseReg(); |
5450 |
getGlobalBaseReg(); |
| 5451 |
} |
5451 |
} |
| 5452 |
} |
5452 |
} |
| 5453 |
break; |
5453 |
break; |
| 5454 |
|
5454 |
|
| 5455 |
case PPCISD::GlobalBaseReg: |
5455 |
case PPCISD::GlobalBaseReg: |
| 5456 |
ReplaceNode(N, getGlobalBaseReg()); |
5456 |
ReplaceNode(N, getGlobalBaseReg()); |
| 5457 |
return; |
5457 |
return; |
| 5458 |
|
5458 |
|
| 5459 |
case ISD::FrameIndex: |
5459 |
case ISD::FrameIndex: |
| 5460 |
selectFrameIndex(N, N); |
5460 |
selectFrameIndex(N, N); |
| 5461 |
return; |
5461 |
return; |
| 5462 |
|
5462 |
|
| 5463 |
case PPCISD::MFOCRF: { |
5463 |
case PPCISD::MFOCRF: { |
| 5464 |
SDValue InGlue = N->getOperand(1); |
5464 |
SDValue InGlue = N->getOperand(1); |
| 5465 |
ReplaceNode(N, CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, |
5465 |
ReplaceNode(N, CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, |
| 5466 |
N->getOperand(0), InGlue)); |
5466 |
N->getOperand(0), InGlue)); |
| 5467 |
return; |
5467 |
return; |
| 5468 |
} |
5468 |
} |
| 5469 |
|
5469 |
|
| 5470 |
case PPCISD::READ_TIME_BASE: |
5470 |
case PPCISD::READ_TIME_BASE: |
| 5471 |
ReplaceNode(N, CurDAG->getMachineNode(PPC::ReadTB, dl, MVT::i32, MVT::i32, |
5471 |
ReplaceNode(N, CurDAG->getMachineNode(PPC::ReadTB, dl, MVT::i32, MVT::i32, |
| 5472 |
MVT::Other, N->getOperand(0))); |
5472 |
MVT::Other, N->getOperand(0))); |
| 5473 |
return; |
5473 |
return; |
| 5474 |
|
5474 |
|
| 5475 |
case PPCISD::SRA_ADDZE: { |
5475 |
case PPCISD::SRA_ADDZE: { |
| 5476 |
SDValue N0 = N->getOperand(0); |
5476 |
SDValue N0 = N->getOperand(0); |
| 5477 |
SDValue ShiftAmt = |
5477 |
SDValue ShiftAmt = |
| 5478 |
CurDAG->getTargetConstant(*cast(N->getOperand(1))-> |
5478 |
CurDAG->getTargetConstant(*cast(N->getOperand(1))-> |
| 5479 |
getConstantIntValue(), dl, |
5479 |
getConstantIntValue(), dl, |
| 5480 |
N->getValueType(0)); |
5480 |
N->getValueType(0)); |
| 5481 |
if (N->getValueType(0) == MVT::i64) { |
5481 |
if (N->getValueType(0) == MVT::i64) { |
| 5482 |
SDNode *Op = |
5482 |
SDNode *Op = |
| 5483 |
CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, MVT::Glue, |
5483 |
CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, MVT::Glue, |
| 5484 |
N0, ShiftAmt); |
5484 |
N0, ShiftAmt); |
| 5485 |
CurDAG->SelectNodeTo(N, PPC::ADDZE8, MVT::i64, SDValue(Op, 0), |
5485 |
CurDAG->SelectNodeTo(N, PPC::ADDZE8, MVT::i64, SDValue(Op, 0), |
| 5486 |
SDValue(Op, 1)); |
5486 |
SDValue(Op, 1)); |
| 5487 |
return; |
5487 |
return; |
| 5488 |
} else { |
5488 |
} else { |
| 5489 |
assert(N->getValueType(0) == MVT::i32 && |
5489 |
assert(N->getValueType(0) == MVT::i32 && |
| 5490 |
"Expecting i64 or i32 in PPCISD::SRA_ADDZE"); |
5490 |
"Expecting i64 or i32 in PPCISD::SRA_ADDZE"); |
| 5491 |
SDNode *Op = |
5491 |
SDNode *Op = |
| 5492 |
CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, MVT::Glue, |
5492 |
CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, MVT::Glue, |
| 5493 |
N0, ShiftAmt); |
5493 |
N0, ShiftAmt); |
| 5494 |
CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, SDValue(Op, 0), |
5494 |
CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, SDValue(Op, 0), |
| 5495 |
SDValue(Op, 1)); |
5495 |
SDValue(Op, 1)); |
| 5496 |
return; |
5496 |
return; |
| 5497 |
} |
5497 |
} |
| 5498 |
} |
5498 |
} |
| 5499 |
|
5499 |
|
| 5500 |
case ISD::STORE: { |
5500 |
case ISD::STORE: { |
| 5501 |
// Change TLS initial-exec (or TLS local-exec on AIX) D-form stores to |
5501 |
// Change TLS initial-exec (or TLS local-exec on AIX) D-form stores to |
| 5502 |
// X-form stores. |
5502 |
// X-form stores. |
| 5503 |
StoreSDNode *ST = cast(N); |
5503 |
StoreSDNode *ST = cast(N); |
| 5504 |
if (EnableTLSOpt && (Subtarget->isELFv2ABI() || Subtarget->isAIXABI()) && |
5504 |
if (EnableTLSOpt && (Subtarget->isELFv2ABI() || Subtarget->isAIXABI()) && |
| 5505 |
ST->getAddressingMode() != ISD::PRE_INC) |
5505 |
ST->getAddressingMode() != ISD::PRE_INC) |
| 5506 |
if (tryTLSXFormStore(ST)) |
5506 |
if (tryTLSXFormStore(ST)) |
| 5507 |
return; |
5507 |
return; |
| 5508 |
break; |
5508 |
break; |
| 5509 |
} |
5509 |
} |
| 5510 |
case ISD::LOAD: { |
5510 |
case ISD::LOAD: { |
| 5511 |
// Handle preincrement loads. |
5511 |
// Handle preincrement loads. |
| 5512 |
LoadSDNode *LD = cast(N); |
5512 |
LoadSDNode *LD = cast(N); |
| 5513 |
EVT LoadedVT = LD->getMemoryVT(); |
5513 |
EVT LoadedVT = LD->getMemoryVT(); |
| 5514 |
|
5514 |
|
| 5515 |
// Normal loads are handled by code generated from the .td file. |
5515 |
// Normal loads are handled by code generated from the .td file. |
| 5516 |
if (LD->getAddressingMode() != ISD::PRE_INC) { |
5516 |
if (LD->getAddressingMode() != ISD::PRE_INC) { |
| 5517 |
// Change TLS initial-exec (or TLS local-exec on AIX) D-form loads to |
5517 |
// Change TLS initial-exec (or TLS local-exec on AIX) D-form loads to |
| 5518 |
// X-form loads. |
5518 |
// X-form loads. |
| 5519 |
if (EnableTLSOpt && (Subtarget->isELFv2ABI() || Subtarget->isAIXABI())) |
5519 |
if (EnableTLSOpt && (Subtarget->isELFv2ABI() || Subtarget->isAIXABI())) |
| 5520 |
if (tryTLSXFormLoad(LD)) |
5520 |
if (tryTLSXFormLoad(LD)) |
| 5521 |
return; |
5521 |
return; |
| 5522 |
break; |
5522 |
break; |
| 5523 |
} |
5523 |
} |
| 5524 |
|
5524 |
|
| 5525 |
SDValue Offset = LD->getOffset(); |
5525 |
SDValue Offset = LD->getOffset(); |
| 5526 |
if (Offset.getOpcode() == ISD::TargetConstant || |
5526 |
if (Offset.getOpcode() == ISD::TargetConstant || |
| 5527 |
Offset.getOpcode() == ISD::TargetGlobalAddress) { |
5527 |
Offset.getOpcode() == ISD::TargetGlobalAddress) { |
| 5528 |
|
5528 |
|
| 5529 |
unsigned Opcode; |
5529 |
unsigned Opcode; |
| 5530 |
bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; |
5530 |
bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; |
| 5531 |
if (LD->getValueType(0) != MVT::i64) { |
5531 |
if (LD->getValueType(0) != MVT::i64) { |
| 5532 |
// Handle PPC32 integer and normal FP loads. |
5532 |
// Handle PPC32 integer and normal FP loads. |
| 5533 |
assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"); |
5533 |
assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"); |
| 5534 |
switch (LoadedVT.getSimpleVT().SimpleTy) { |
5534 |
switch (LoadedVT.getSimpleVT().SimpleTy) { |
| 5535 |
default: llvm_unreachable("Invalid PPC load type!"); |
5535 |
default: llvm_unreachable("Invalid PPC load type!"); |
| 5536 |
case MVT::f64: Opcode = PPC::LFDU; break; |
5536 |
case MVT::f64: Opcode = PPC::LFDU; break; |
| 5537 |
case MVT::f32: Opcode = PPC::LFSU; break; |
5537 |
case MVT::f32: Opcode = PPC::LFSU; break; |
| 5538 |
case MVT::i32: Opcode = PPC::LWZU; break; |
5538 |
case MVT::i32: Opcode = PPC::LWZU; break; |
| 5539 |
case MVT::i16: Opcode = isSExt ? PPC::LHAU : PPC::LHZU; break; |
5539 |
case MVT::i16: Opcode = isSExt ? PPC::LHAU : PPC::LHZU; break; |
| 5540 |
case MVT::i1: |
5540 |
case MVT::i1: |
| 5541 |
case MVT::i8: Opcode = PPC::LBZU; break; |
5541 |
case MVT::i8: Opcode = PPC::LBZU; break; |
| 5542 |
} |
5542 |
} |
| 5543 |
} else { |
5543 |
} else { |
| 5544 |
assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!"); |
5544 |
assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!"); |
| 5545 |
assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"); |
5545 |
assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"); |
| 5546 |
switch (LoadedVT.getSimpleVT().SimpleTy) { |
5546 |
switch (LoadedVT.getSimpleVT().SimpleTy) { |
| 5547 |
default: llvm_unreachable("Invalid PPC load type!"); |
5547 |
default: llvm_unreachable("Invalid PPC load type!"); |
| 5548 |
case MVT::i64: Opcode = PPC::LDU; break; |
5548 |
case MVT::i64: Opcode = PPC::LDU; break; |
| 5549 |
case MVT::i32: Opcode = PPC::LWZU8; break; |
5549 |
case MVT::i32: Opcode = PPC::LWZU8; break; |
| 5550 |
case MVT::i16: Opcode = isSExt ? PPC::LHAU8 : PPC::LHZU8; break; |
5550 |
case MVT::i16: Opcode = isSExt ? PPC::LHAU8 : PPC::LHZU8; break; |
| 5551 |
case MVT::i1: |
5551 |
case MVT::i1: |
| 5552 |
case MVT::i8: Opcode = PPC::LBZU8; break; |
5552 |
case MVT::i8: Opcode = PPC::LBZU8; break; |
| 5553 |
} |
5553 |
} |
| 5554 |
} |
5554 |
} |
| 5555 |
|
5555 |
|
| 5556 |
SDValue Chain = LD->getChain(); |
5556 |
SDValue Chain = LD->getChain(); |
| 5557 |
SDValue Base = LD->getBasePtr(); |
5557 |
SDValue Base = LD->getBasePtr(); |
| 5558 |
SDValue Ops[] = { Offset, Base, Chain }; |
5558 |
SDValue Ops[] = { Offset, Base, Chain }; |
| 5559 |
SDNode *MN = CurDAG->getMachineNode( |
5559 |
SDNode *MN = CurDAG->getMachineNode( |
| 5560 |
Opcode, dl, LD->getValueType(0), |
5560 |
Opcode, dl, LD->getValueType(0), |
| 5561 |
PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops); |
5561 |
PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops); |
| 5562 |
transferMemOperands(N, MN); |
5562 |
transferMemOperands(N, MN); |
| 5563 |
ReplaceNode(N, MN); |
5563 |
ReplaceNode(N, MN); |
| 5564 |
return; |
5564 |
return; |
| 5565 |
} else { |
5565 |
} else { |
| 5566 |
unsigned Opcode; |
5566 |
unsigned Opcode; |
| 5567 |
bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; |
5567 |
bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; |
| 5568 |
if (LD->getValueType(0) != MVT::i64) { |
5568 |
if (LD->getValueType(0) != MVT::i64) { |
| 5569 |
// Handle PPC32 integer and normal FP loads. |
5569 |
// Handle PPC32 integer and normal FP loads. |
| 5570 |
assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"); |
5570 |
assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"); |
| 5571 |
switch (LoadedVT.getSimpleVT().SimpleTy) { |
5571 |
switch (LoadedVT.getSimpleVT().SimpleTy) { |
| 5572 |
default: llvm_unreachable("Invalid PPC load type!"); |
5572 |
default: llvm_unreachable("Invalid PPC load type!"); |
| 5573 |
case MVT::f64: Opcode = PPC::LFDUX; break; |
5573 |
case MVT::f64: Opcode = PPC::LFDUX; break; |
| 5574 |
case MVT::f32: Opcode = PPC::LFSUX; break; |
5574 |
case MVT::f32: Opcode = PPC::LFSUX; break; |
| 5575 |
case MVT::i32: Opcode = PPC::LWZUX; break; |
5575 |
case MVT::i32: Opcode = PPC::LWZUX; break; |
| 5576 |
case MVT::i16: Opcode = isSExt ? PPC::LHAUX : PPC::LHZUX; break; |
5576 |
case MVT::i16: Opcode = isSExt ? PPC::LHAUX : PPC::LHZUX; break; |
| 5577 |
case MVT::i1: |
5577 |
case MVT::i1: |
| 5578 |
case MVT::i8: Opcode = PPC::LBZUX; break; |
5578 |
case MVT::i8: Opcode = PPC::LBZUX; break; |
| 5579 |
} |
5579 |
} |
| 5580 |
} else { |
5580 |
} else { |
| 5581 |
assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!"); |
5581 |
assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!"); |
| 5582 |
assert((!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) && |
5582 |
assert((!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) && |
| 5583 |
"Invalid sext update load"); |
5583 |
"Invalid sext update load"); |
| 5584 |
switch (LoadedVT.getSimpleVT().SimpleTy) { |
5584 |
switch (LoadedVT.getSimpleVT().SimpleTy) { |
| 5585 |
default: llvm_unreachable("Invalid PPC load type!"); |
5585 |
default: llvm_unreachable("Invalid PPC load type!"); |
| 5586 |
case MVT::i64: Opcode = PPC::LDUX; break; |
5586 |
case MVT::i64: Opcode = PPC::LDUX; break; |
| 5587 |
case MVT::i32: Opcode = isSExt ? PPC::LWAUX : PPC::LWZUX8; break; |
5587 |
case MVT::i32: Opcode = isSExt ? PPC::LWAUX : PPC::LWZUX8; break; |
| 5588 |
case MVT::i16: Opcode = isSExt ? PPC::LHAUX8 : PPC::LHZUX8; break; |
5588 |
case MVT::i16: Opcode = isSExt ? PPC::LHAUX8 : PPC::LHZUX8; break; |
| 5589 |
case MVT::i1: |
5589 |
case MVT::i1: |
| 5590 |
case MVT::i8: Opcode = PPC::LBZUX8; break; |
5590 |
case MVT::i8: Opcode = PPC::LBZUX8; break; |
| 5591 |
} |
5591 |
} |
| 5592 |
} |
5592 |
} |
| 5593 |
|
5593 |
|
| 5594 |
SDValue Chain = LD->getChain(); |
5594 |
SDValue Chain = LD->getChain(); |
| 5595 |
SDValue Base = LD->getBasePtr(); |
5595 |
SDValue Base = LD->getBasePtr(); |
| 5596 |
SDValue Ops[] = { Base, Offset, Chain }; |
5596 |
SDValue Ops[] = { Base, Offset, Chain }; |
| 5597 |
SDNode *MN = CurDAG->getMachineNode( |
5597 |
SDNode *MN = CurDAG->getMachineNode( |
| 5598 |
Opcode, dl, LD->getValueType(0), |
5598 |
Opcode, dl, LD->getValueType(0), |
| 5599 |
PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops); |
5599 |
PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops); |
| 5600 |
transferMemOperands(N, MN); |
5600 |
transferMemOperands(N, MN); |
| 5601 |
ReplaceNode(N, MN); |
5601 |
ReplaceNode(N, MN); |
| 5602 |
return; |
5602 |
return; |
| 5603 |
} |
5603 |
} |
| 5604 |
} |
5604 |
} |
| 5605 |
|
5605 |
|
| 5606 |
case ISD::AND: |
5606 |
case ISD::AND: |
| 5607 |
// If this is an 'and' with a mask, try to emit rlwinm/rldicl/rldicr |
5607 |
// If this is an 'and' with a mask, try to emit rlwinm/rldicl/rldicr |
| 5608 |
if (tryAsSingleRLWINM(N) || tryAsSingleRLWIMI(N) || tryAsSingleRLDICL(N) || |
5608 |
if (tryAsSingleRLWINM(N) || tryAsSingleRLWIMI(N) || tryAsSingleRLDICL(N) || |
| 5609 |
tryAsSingleRLDICR(N) || tryAsSingleRLWINM8(N) || tryAsPairOfRLDICL(N)) |
5609 |
tryAsSingleRLDICR(N) || tryAsSingleRLWINM8(N) || tryAsPairOfRLDICL(N)) |
| 5610 |
return; |
5610 |
return; |
| 5611 |
|
5611 |
|
| 5612 |
// Other cases are autogenerated. |
5612 |
// Other cases are autogenerated. |
| 5613 |
break; |
5613 |
break; |
| 5614 |
case ISD::OR: { |
5614 |
case ISD::OR: { |
| 5615 |
if (N->getValueType(0) == MVT::i32) |
5615 |
if (N->getValueType(0) == MVT::i32) |
| 5616 |
if (tryBitfieldInsert(N)) |
5616 |
if (tryBitfieldInsert(N)) |
| 5617 |
return; |
5617 |
return; |
| 5618 |
|
5618 |
|
| 5619 |
int16_t Imm; |
5619 |
int16_t Imm; |
| 5620 |
if (N->getOperand(0)->getOpcode() == ISD::FrameIndex && |
5620 |
if (N->getOperand(0)->getOpcode() == ISD::FrameIndex && |
| 5621 |
isIntS16Immediate(N->getOperand(1), Imm)) { |
5621 |
isIntS16Immediate(N->getOperand(1), Imm)) { |
| 5622 |
KnownBits LHSKnown = CurDAG->computeKnownBits(N->getOperand(0)); |
5622 |
KnownBits LHSKnown = CurDAG->computeKnownBits(N->getOperand(0)); |
| 5623 |
|
5623 |
|
| 5624 |
// If this is equivalent to an add, then we can fold it with the |
5624 |
// If this is equivalent to an add, then we can fold it with the |
| 5625 |
// FrameIndex calculation. |
5625 |
// FrameIndex calculation. |
| 5626 |
if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)Imm) == ~0ULL) { |
5626 |
if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)Imm) == ~0ULL) { |
| 5627 |
selectFrameIndex(N, N->getOperand(0).getNode(), (int64_t)Imm); |
5627 |
selectFrameIndex(N, N->getOperand(0).getNode(), (int64_t)Imm); |
| 5628 |
return; |
5628 |
return; |
| 5629 |
} |
5629 |
} |
| 5630 |
} |
5630 |
} |
| 5631 |
|
5631 |
|
| 5632 |
// If this is 'or' against an imm with consecutive ones and both sides zero, |
5632 |
// If this is 'or' against an imm with consecutive ones and both sides zero, |
| 5633 |
// try to emit rldimi |
5633 |
// try to emit rldimi |
| 5634 |
if (tryAsSingleRLDIMI(N)) |
5634 |
if (tryAsSingleRLDIMI(N)) |
| 5635 |
return; |
5635 |
return; |
| 5636 |
|
5636 |
|
| 5637 |
// OR with a 32-bit immediate can be handled by ori + oris |
5637 |
// OR with a 32-bit immediate can be handled by ori + oris |
| 5638 |
// without creating an immediate in a GPR. |
5638 |
// without creating an immediate in a GPR. |
| 5639 |
uint64_t Imm64 = 0; |
5639 |
uint64_t Imm64 = 0; |
| 5640 |
bool IsPPC64 = Subtarget->isPPC64(); |
5640 |
bool IsPPC64 = Subtarget->isPPC64(); |
| 5641 |
if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) && |
5641 |
if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) && |
| 5642 |
(Imm64 & ~0xFFFFFFFFuLL) == 0) { |
5642 |
(Imm64 & ~0xFFFFFFFFuLL) == 0) { |
| 5643 |
// If ImmHi (ImmHi) is zero, only one ori (oris) is generated later. |
5643 |
// If ImmHi (ImmHi) is zero, only one ori (oris) is generated later. |
| 5644 |
uint64_t ImmHi = Imm64 >> 16; |
5644 |
uint64_t ImmHi = Imm64 >> 16; |
| 5645 |
uint64_t ImmLo = Imm64 & 0xFFFF; |
5645 |
uint64_t ImmLo = Imm64 & 0xFFFF; |
| 5646 |
if (ImmHi != 0 && ImmLo != 0) { |
5646 |
if (ImmHi != 0 && ImmLo != 0) { |
| 5647 |
SDNode *Lo = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, |
5647 |
SDNode *Lo = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, |
| 5648 |
N->getOperand(0), |
5648 |
N->getOperand(0), |
| 5649 |
getI16Imm(ImmLo, dl)); |
5649 |
getI16Imm(ImmLo, dl)); |
| 5650 |
SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(ImmHi, dl)}; |
5650 |
SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(ImmHi, dl)}; |
| 5651 |
CurDAG->SelectNodeTo(N, PPC::ORIS8, MVT::i64, Ops1); |
5651 |
CurDAG->SelectNodeTo(N, PPC::ORIS8, MVT::i64, Ops1); |
| 5652 |
return; |
5652 |
return; |
| 5653 |
} |
5653 |
} |
| 5654 |
} |
5654 |
} |
| 5655 |
|
5655 |
|
| 5656 |
// Other cases are autogenerated. |
5656 |
// Other cases are autogenerated. |
| 5657 |
break; |
5657 |
break; |
| 5658 |
} |
5658 |
} |
| 5659 |
case ISD::XOR: { |
5659 |
case ISD::XOR: { |
| 5660 |
// XOR with a 32-bit immediate can be handled by xori + xoris |
5660 |
// XOR with a 32-bit immediate can be handled by xori + xoris |
| 5661 |
// without creating an immediate in a GPR. |
5661 |
// without creating an immediate in a GPR. |
| 5662 |
uint64_t Imm64 = 0; |
5662 |
uint64_t Imm64 = 0; |
| 5663 |
bool IsPPC64 = Subtarget->isPPC64(); |
5663 |
bool IsPPC64 = Subtarget->isPPC64(); |
| 5664 |
if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) && |
5664 |
if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) && |
| 5665 |
(Imm64 & ~0xFFFFFFFFuLL) == 0) { |
5665 |
(Imm64 & ~0xFFFFFFFFuLL) == 0) { |
| 5666 |
// If ImmHi (ImmHi) is zero, only one xori (xoris) is generated later. |
5666 |
// If ImmHi (ImmHi) is zero, only one xori (xoris) is generated later. |
| 5667 |
uint64_t ImmHi = Imm64 >> 16; |
5667 |
uint64_t ImmHi = Imm64 >> 16; |
| 5668 |
uint64_t ImmLo = Imm64 & 0xFFFF; |
5668 |
uint64_t ImmLo = Imm64 & 0xFFFF; |
| 5669 |
if (ImmHi != 0 && ImmLo != 0) { |
5669 |
if (ImmHi != 0 && ImmLo != 0) { |
| 5670 |
SDNode *Lo = CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, |
5670 |
SDNode *Lo = CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, |
| 5671 |
N->getOperand(0), |
5671 |
N->getOperand(0), |
| 5672 |
getI16Imm(ImmLo, dl)); |
5672 |
getI16Imm(ImmLo, dl)); |
| 5673 |
SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(ImmHi, dl)}; |
5673 |
SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(ImmHi, dl)}; |
| 5674 |
CurDAG->SelectNodeTo(N, PPC::XORIS8, MVT::i64, Ops1); |
5674 |
CurDAG->SelectNodeTo(N, PPC::XORIS8, MVT::i64, Ops1); |
| 5675 |
return; |
5675 |
return; |
| 5676 |
} |
5676 |
} |
| 5677 |
} |
5677 |
} |
| 5678 |
|
5678 |
|
| 5679 |
break; |
5679 |
break; |
| 5680 |
} |
5680 |
} |
| 5681 |
case ISD::ADD: { |
5681 |
case ISD::ADD: { |
| 5682 |
int16_t Imm; |
5682 |
int16_t Imm; |
| 5683 |
if (N->getOperand(0)->getOpcode() == ISD::FrameIndex && |
5683 |
if (N->getOperand(0)->getOpcode() == ISD::FrameIndex && |
| 5684 |
isIntS16Immediate(N->getOperand(1), Imm)) { |
5684 |
isIntS16Immediate(N->getOperand(1), Imm)) { |
| 5685 |
selectFrameIndex(N, N->getOperand(0).getNode(), (int64_t)Imm); |
5685 |
selectFrameIndex(N, N->getOperand(0).getNode(), (int64_t)Imm); |
| 5686 |
return; |
5686 |
return; |
| 5687 |
} |
5687 |
} |
| 5688 |
|
5688 |
|
| 5689 |
break; |
5689 |
break; |
| 5690 |
} |
5690 |
} |
| 5691 |
case ISD::SHL: { |
5691 |
case ISD::SHL: { |
| 5692 |
unsigned Imm, SH, MB, ME; |
5692 |
unsigned Imm, SH, MB, ME; |
| 5693 |
if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) && |
5693 |
if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) && |
| 5694 |
isRotateAndMask(N, Imm, true, SH, MB, ME)) { |
5694 |
isRotateAndMask(N, Imm, true, SH, MB, ME)) { |
| 5695 |
SDValue Ops[] = { N->getOperand(0).getOperand(0), |
5695 |
SDValue Ops[] = { N->getOperand(0).getOperand(0), |
| 5696 |
getI32Imm(SH, dl), getI32Imm(MB, dl), |
5696 |
getI32Imm(SH, dl), getI32Imm(MB, dl), |
| 5697 |
getI32Imm(ME, dl) }; |
5697 |
getI32Imm(ME, dl) }; |
| 5698 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
5698 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
| 5699 |
return; |
5699 |
return; |
| 5700 |
} |
5700 |
} |
| 5701 |
|
5701 |
|
| 5702 |
// Other cases are autogenerated. |
5702 |
// Other cases are autogenerated. |
| 5703 |
break; |
5703 |
break; |
| 5704 |
} |
5704 |
} |
| 5705 |
case ISD::SRL: { |
5705 |
case ISD::SRL: { |
| 5706 |
unsigned Imm, SH, MB, ME; |
5706 |
unsigned Imm, SH, MB, ME; |
| 5707 |
if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) && |
5707 |
if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) && |
| 5708 |
isRotateAndMask(N, Imm, true, SH, MB, ME)) { |
5708 |
isRotateAndMask(N, Imm, true, SH, MB, ME)) { |
| 5709 |
SDValue Ops[] = { N->getOperand(0).getOperand(0), |
5709 |
SDValue Ops[] = { N->getOperand(0).getOperand(0), |
| 5710 |
getI32Imm(SH, dl), getI32Imm(MB, dl), |
5710 |
getI32Imm(SH, dl), getI32Imm(MB, dl), |
| 5711 |
getI32Imm(ME, dl) }; |
5711 |
getI32Imm(ME, dl) }; |
| 5712 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
5712 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
| 5713 |
return; |
5713 |
return; |
| 5714 |
} |
5714 |
} |
| 5715 |
|
5715 |
|
| 5716 |
// Other cases are autogenerated. |
5716 |
// Other cases are autogenerated. |
| 5717 |
break; |
5717 |
break; |
| 5718 |
} |
5718 |
} |
| 5719 |
case ISD::MUL: { |
5719 |
case ISD::MUL: { |
| 5720 |
SDValue Op1 = N->getOperand(1); |
5720 |
SDValue Op1 = N->getOperand(1); |
| 5721 |
if (Op1.getOpcode() != ISD::Constant || |
5721 |
if (Op1.getOpcode() != ISD::Constant || |
| 5722 |
(Op1.getValueType() != MVT::i64 && Op1.getValueType() != MVT::i32)) |
5722 |
(Op1.getValueType() != MVT::i64 && Op1.getValueType() != MVT::i32)) |
| 5723 |
break; |
5723 |
break; |
| 5724 |
|
5724 |
|
| 5725 |
// If the multiplier fits int16, we can handle it with mulli. |
5725 |
// If the multiplier fits int16, we can handle it with mulli. |
| 5726 |
int64_t Imm = cast(Op1)->getZExtValue(); |
5726 |
int64_t Imm = cast(Op1)->getZExtValue(); |
| 5727 |
unsigned Shift = llvm::countr_zero(Imm); |
5727 |
unsigned Shift = llvm::countr_zero(Imm); |
| 5728 |
if (isInt<16>(Imm) || !Shift) |
5728 |
if (isInt<16>(Imm) || !Shift) |
| 5729 |
break; |
5729 |
break; |
| 5730 |
|
5730 |
|
| 5731 |
// If the shifted value fits int16, we can do this transformation: |
5731 |
// If the shifted value fits int16, we can do this transformation: |
| 5732 |
// (mul X, c1 << c2) -> (rldicr (mulli X, c1) c2). We do this in ISEL due to |
5732 |
// (mul X, c1 << c2) -> (rldicr (mulli X, c1) c2). We do this in ISEL due to |
| 5733 |
// DAGCombiner prefers (shl (mul X, c1), c2) -> (mul X, c1 << c2). |
5733 |
// DAGCombiner prefers (shl (mul X, c1), c2) -> (mul X, c1 << c2). |
| 5734 |
uint64_t ImmSh = Imm >> Shift; |
5734 |
uint64_t ImmSh = Imm >> Shift; |
| 5735 |
if (!isInt<16>(ImmSh)) |
5735 |
if (!isInt<16>(ImmSh)) |
| 5736 |
break; |
5736 |
break; |
| 5737 |
|
5737 |
|
| 5738 |
uint64_t SextImm = SignExtend64(ImmSh & 0xFFFF, 16); |
5738 |
uint64_t SextImm = SignExtend64(ImmSh & 0xFFFF, 16); |
| 5739 |
if (Op1.getValueType() == MVT::i64) { |
5739 |
if (Op1.getValueType() == MVT::i64) { |
| 5740 |
SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64); |
5740 |
SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64); |
| 5741 |
SDNode *MulNode = CurDAG->getMachineNode(PPC::MULLI8, dl, MVT::i64, |
5741 |
SDNode *MulNode = CurDAG->getMachineNode(PPC::MULLI8, dl, MVT::i64, |
| 5742 |
N->getOperand(0), SDImm); |
5742 |
N->getOperand(0), SDImm); |
| 5743 |
|
5743 |
|
| 5744 |
SDValue Ops[] = {SDValue(MulNode, 0), getI32Imm(Shift, dl), |
5744 |
SDValue Ops[] = {SDValue(MulNode, 0), getI32Imm(Shift, dl), |
| 5745 |
getI32Imm(63 - Shift, dl)}; |
5745 |
getI32Imm(63 - Shift, dl)}; |
| 5746 |
CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, Ops); |
5746 |
CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, Ops); |
| 5747 |
return; |
5747 |
return; |
| 5748 |
} else { |
5748 |
} else { |
| 5749 |
SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i32); |
5749 |
SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i32); |
| 5750 |
SDNode *MulNode = CurDAG->getMachineNode(PPC::MULLI, dl, MVT::i32, |
5750 |
SDNode *MulNode = CurDAG->getMachineNode(PPC::MULLI, dl, MVT::i32, |
| 5751 |
N->getOperand(0), SDImm); |
5751 |
N->getOperand(0), SDImm); |
| 5752 |
|
5752 |
|
| 5753 |
SDValue Ops[] = {SDValue(MulNode, 0), getI32Imm(Shift, dl), |
5753 |
SDValue Ops[] = {SDValue(MulNode, 0), getI32Imm(Shift, dl), |
| 5754 |
getI32Imm(0, dl), getI32Imm(31 - Shift, dl)}; |
5754 |
getI32Imm(0, dl), getI32Imm(31 - Shift, dl)}; |
| 5755 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
5755 |
CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
| 5756 |
return; |
5756 |
return; |
| 5757 |
} |
5757 |
} |
| 5758 |
break; |
5758 |
break; |
| 5759 |
} |
5759 |
} |
| 5760 |
// FIXME: Remove this once the ANDI glue bug is fixed: |
5760 |
// FIXME: Remove this once the ANDI glue bug is fixed: |
| 5761 |
case PPCISD::ANDI_rec_1_EQ_BIT: |
5761 |
case PPCISD::ANDI_rec_1_EQ_BIT: |
| 5762 |
case PPCISD::ANDI_rec_1_GT_BIT: { |
5762 |
case PPCISD::ANDI_rec_1_GT_BIT: { |
| 5763 |
if (!ANDIGlueBug) |
5763 |
if (!ANDIGlueBug) |
| 5764 |
break; |
5764 |
break; |
| 5765 |
|
5765 |
|
| 5766 |
EVT InVT = N->getOperand(0).getValueType(); |
5766 |
EVT InVT = N->getOperand(0).getValueType(); |
| 5767 |
assert((InVT == MVT::i64 || InVT == MVT::i32) && |
5767 |
assert((InVT == MVT::i64 || InVT == MVT::i32) && |
| 5768 |
"Invalid input type for ANDI_rec_1_EQ_BIT"); |
5768 |
"Invalid input type for ANDI_rec_1_EQ_BIT"); |
| 5769 |
|
5769 |
|
| 5770 |
unsigned Opcode = (InVT == MVT::i64) ? PPC::ANDI8_rec : PPC::ANDI_rec; |
5770 |
unsigned Opcode = (InVT == MVT::i64) ? PPC::ANDI8_rec : PPC::ANDI_rec; |
| 5771 |
SDValue AndI(CurDAG->getMachineNode(Opcode, dl, InVT, MVT::Glue, |
5771 |
SDValue AndI(CurDAG->getMachineNode(Opcode, dl, InVT, MVT::Glue, |
| 5772 |
N->getOperand(0), |
5772 |
N->getOperand(0), |
| 5773 |
CurDAG->getTargetConstant(1, dl, InVT)), |
5773 |
CurDAG->getTargetConstant(1, dl, InVT)), |
| 5774 |
0); |
5774 |
0); |
| 5775 |
SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32); |
5775 |
SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32); |
| 5776 |
SDValue SRIdxVal = CurDAG->getTargetConstant( |
5776 |
SDValue SRIdxVal = CurDAG->getTargetConstant( |
| 5777 |
N->getOpcode() == PPCISD::ANDI_rec_1_EQ_BIT ? PPC::sub_eq : PPC::sub_gt, |
5777 |
N->getOpcode() == PPCISD::ANDI_rec_1_EQ_BIT ? PPC::sub_eq : PPC::sub_gt, |
| 5778 |
dl, MVT::i32); |
5778 |
dl, MVT::i32); |
| 5779 |
|
5779 |
|
| 5780 |
CurDAG->SelectNodeTo(N, TargetOpcode::EXTRACT_SUBREG, MVT::i1, CR0Reg, |
5780 |
CurDAG->SelectNodeTo(N, TargetOpcode::EXTRACT_SUBREG, MVT::i1, CR0Reg, |
| 5781 |
SRIdxVal, SDValue(AndI.getNode(), 1) /* glue */); |
5781 |
SRIdxVal, SDValue(AndI.getNode(), 1) /* glue */); |
| 5782 |
return; |
5782 |
return; |
| 5783 |
} |
5783 |
} |
| 5784 |
case ISD::SELECT_CC: { |
5784 |
case ISD::SELECT_CC: { |
| 5785 |
ISD::CondCode CC = cast(N->getOperand(4))->get(); |
5785 |
ISD::CondCode CC = cast(N->getOperand(4))->get(); |
| 5786 |
EVT PtrVT = |
5786 |
EVT PtrVT = |
| 5787 |
CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout()); |
5787 |
CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout()); |
| 5788 |
bool isPPC64 = (PtrVT == MVT::i64); |
5788 |
bool isPPC64 = (PtrVT == MVT::i64); |
| 5789 |
|
5789 |
|
| 5790 |
// If this is a select of i1 operands, we'll pattern match it. |
5790 |
// If this is a select of i1 operands, we'll pattern match it. |
| 5791 |
if (Subtarget->useCRBits() && N->getOperand(0).getValueType() == MVT::i1) |
5791 |
if (Subtarget->useCRBits() && N->getOperand(0).getValueType() == MVT::i1) |
| 5792 |
break; |
5792 |
break; |
| 5793 |
|
5793 |
|
| 5794 |
if (Subtarget->isISA3_0() && Subtarget->isPPC64()) { |
5794 |
if (Subtarget->isISA3_0() && Subtarget->isPPC64()) { |
| 5795 |
bool NeedSwapOps = false; |
5795 |
bool NeedSwapOps = false; |
| 5796 |
bool IsUnCmp = false; |
5796 |
bool IsUnCmp = false; |
| 5797 |
if (mayUseP9Setb(N, CC, CurDAG, NeedSwapOps, IsUnCmp)) { |
5797 |
if (mayUseP9Setb(N, CC, CurDAG, NeedSwapOps, IsUnCmp)) { |
| 5798 |
SDValue LHS = N->getOperand(0); |
5798 |
SDValue LHS = N->getOperand(0); |
| 5799 |
SDValue RHS = N->getOperand(1); |
5799 |
SDValue RHS = N->getOperand(1); |
| 5800 |
if (NeedSwapOps) |
5800 |
if (NeedSwapOps) |
| 5801 |
std::swap(LHS, RHS); |
5801 |
std::swap(LHS, RHS); |
| 5802 |
|
5802 |
|
| 5803 |
// Make use of SelectCC to generate the comparison to set CR bits, for |
5803 |
// Make use of SelectCC to generate the comparison to set CR bits, for |
| 5804 |
// equality comparisons having one literal operand, SelectCC probably |
5804 |
// equality comparisons having one literal operand, SelectCC probably |
| 5805 |
// doesn't need to materialize the whole literal and just use xoris to |
5805 |
// doesn't need to materialize the whole literal and just use xoris to |
| 5806 |
// check it first, it leads the following comparison result can't |
5806 |
// check it first, it leads the following comparison result can't |
| 5807 |
// exactly represent GT/LT relationship. So to avoid this we specify |
5807 |
// exactly represent GT/LT relationship. So to avoid this we specify |
| 5808 |
// SETGT/SETUGT here instead of SETEQ. |
5808 |
// SETGT/SETUGT here instead of SETEQ. |
| 5809 |
SDValue GenCC = |
5809 |
SDValue GenCC = |
| 5810 |
SelectCC(LHS, RHS, IsUnCmp ? ISD::SETUGT : ISD::SETGT, dl); |
5810 |
SelectCC(LHS, RHS, IsUnCmp ? ISD::SETUGT : ISD::SETGT, dl); |
| 5811 |
CurDAG->SelectNodeTo( |
5811 |
CurDAG->SelectNodeTo( |
| 5812 |
N, N->getSimpleValueType(0) == MVT::i64 ? PPC::SETB8 : PPC::SETB, |
5812 |
N, N->getSimpleValueType(0) == MVT::i64 ? PPC::SETB8 : PPC::SETB, |
| 5813 |
N->getValueType(0), GenCC); |
5813 |
N->getValueType(0), GenCC); |
| 5814 |
NumP9Setb++; |
5814 |
NumP9Setb++; |
| 5815 |
return; |
5815 |
return; |
| 5816 |
} |
5816 |
} |
| 5817 |
} |
5817 |
} |
| 5818 |
|
5818 |
|
| 5819 |
// Handle the setcc cases here. select_cc lhs, 0, 1, 0, cc |
5819 |
// Handle the setcc cases here. select_cc lhs, 0, 1, 0, cc |
| 5820 |
if (!isPPC64 && isNullConstant(N->getOperand(1)) && |
5820 |
if (!isPPC64 && isNullConstant(N->getOperand(1)) && |
| 5821 |
isOneConstant(N->getOperand(2)) && isNullConstant(N->getOperand(3)) && |
5821 |
isOneConstant(N->getOperand(2)) && isNullConstant(N->getOperand(3)) && |
| 5822 |
CC == ISD::SETNE && |
5822 |
CC == ISD::SETNE && |
| 5823 |
// FIXME: Implement this optzn for PPC64. |
5823 |
// FIXME: Implement this optzn for PPC64. |
| 5824 |
N->getValueType(0) == MVT::i32) { |
5824 |
N->getValueType(0) == MVT::i32) { |
| 5825 |
SDNode *Tmp = |
5825 |
SDNode *Tmp = |
| 5826 |
CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, |
5826 |
CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, |
| 5827 |
N->getOperand(0), getI32Imm(~0U, dl)); |
5827 |
N->getOperand(0), getI32Imm(~0U, dl)); |
| 5828 |
CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(Tmp, 0), |
5828 |
CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(Tmp, 0), |
| 5829 |
N->getOperand(0), SDValue(Tmp, 1)); |
5829 |
N->getOperand(0), SDValue(Tmp, 1)); |
| 5830 |
return; |
5830 |
return; |
| 5831 |
} |
5831 |
} |
| 5832 |
|
5832 |
|
| 5833 |
SDValue CCReg = SelectCC(N->getOperand(0), N->getOperand(1), CC, dl); |
5833 |
SDValue CCReg = SelectCC(N->getOperand(0), N->getOperand(1), CC, dl); |
| 5834 |
|
5834 |
|
| 5835 |
if (N->getValueType(0) == MVT::i1) { |
5835 |
if (N->getValueType(0) == MVT::i1) { |
| 5836 |
// An i1 select is: (c & t) | (!c & f). |
5836 |
// An i1 select is: (c & t) | (!c & f). |
| 5837 |
bool Inv; |
5837 |
bool Inv; |
| 5838 |
unsigned Idx = getCRIdxForSetCC(CC, Inv); |
5838 |
unsigned Idx = getCRIdxForSetCC(CC, Inv); |
| 5839 |
|
5839 |
|
| 5840 |
unsigned SRI; |
5840 |
unsigned SRI; |
| 5841 |
switch (Idx) { |
5841 |
switch (Idx) { |
| 5842 |
default: llvm_unreachable("Invalid CC index"); |
5842 |
default: llvm_unreachable("Invalid CC index"); |
| 5843 |
case 0: SRI = PPC::sub_lt; break; |
5843 |
case 0: SRI = PPC::sub_lt; break; |
| 5844 |
case 1: SRI = PPC::sub_gt; break; |
5844 |
case 1: SRI = PPC::sub_gt; break; |
| 5845 |
case 2: SRI = PPC::sub_eq; break; |
5845 |
case 2: SRI = PPC::sub_eq; break; |
| 5846 |
case 3: SRI = PPC::sub_un; break; |
5846 |
case 3: SRI = PPC::sub_un; break; |
| 5847 |
} |
5847 |
} |
| 5848 |
|
5848 |
|
| 5849 |
SDValue CCBit = CurDAG->getTargetExtractSubreg(SRI, dl, MVT::i1, CCReg); |
5849 |
SDValue CCBit = CurDAG->getTargetExtractSubreg(SRI, dl, MVT::i1, CCReg); |
| 5850 |
|
5850 |
|
| 5851 |
SDValue NotCCBit(CurDAG->getMachineNode(PPC::CRNOR, dl, MVT::i1, |
5851 |
SDValue NotCCBit(CurDAG->getMachineNode(PPC::CRNOR, dl, MVT::i1, |
| 5852 |
CCBit, CCBit), 0); |
5852 |
CCBit, CCBit), 0); |
| 5853 |
SDValue C = Inv ? NotCCBit : CCBit, |
5853 |
SDValue C = Inv ? NotCCBit : CCBit, |
| 5854 |
NotC = Inv ? CCBit : NotCCBit; |
5854 |
NotC = Inv ? CCBit : NotCCBit; |
| 5855 |
|
5855 |
|
| 5856 |
SDValue CAndT(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1, |
5856 |
SDValue CAndT(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1, |
| 5857 |
C, N->getOperand(2)), 0); |
5857 |
C, N->getOperand(2)), 0); |
| 5858 |
SDValue NotCAndF(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1, |
5858 |
SDValue NotCAndF(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1, |
| 5859 |
NotC, N->getOperand(3)), 0); |
5859 |
NotC, N->getOperand(3)), 0); |
| 5860 |
|
5860 |
|
| 5861 |
CurDAG->SelectNodeTo(N, PPC::CROR, MVT::i1, CAndT, NotCAndF); |
5861 |
CurDAG->SelectNodeTo(N, PPC::CROR, MVT::i1, CAndT, NotCAndF); |
| 5862 |
return; |
5862 |
return; |
| 5863 |
} |
5863 |
} |
| 5864 |
|
5864 |
|
| 5865 |
unsigned BROpc = |
5865 |
unsigned BROpc = |
| 5866 |
getPredicateForSetCC(CC, N->getOperand(0).getValueType(), Subtarget); |
5866 |
getPredicateForSetCC(CC, N->getOperand(0).getValueType(), Subtarget); |
| 5867 |
|
5867 |
|
| 5868 |
unsigned SelectCCOp; |
5868 |
unsigned SelectCCOp; |
| 5869 |
if (N->getValueType(0) == MVT::i32) |
5869 |
if (N->getValueType(0) == MVT::i32) |
| 5870 |
SelectCCOp = PPC::SELECT_CC_I4; |
5870 |
SelectCCOp = PPC::SELECT_CC_I4; |
| 5871 |
else if (N->getValueType(0) == MVT::i64) |
5871 |
else if (N->getValueType(0) == MVT::i64) |
| 5872 |
SelectCCOp = PPC::SELECT_CC_I8; |
5872 |
SelectCCOp = PPC::SELECT_CC_I8; |
| 5873 |
else if (N->getValueType(0) == MVT::f32) { |
5873 |
else if (N->getValueType(0) == MVT::f32) { |
| 5874 |
if (Subtarget->hasP8Vector()) |
5874 |
if (Subtarget->hasP8Vector()) |
| 5875 |
SelectCCOp = PPC::SELECT_CC_VSSRC; |
5875 |
SelectCCOp = PPC::SELECT_CC_VSSRC; |
| 5876 |
else if (Subtarget->hasSPE()) |
5876 |
else if (Subtarget->hasSPE()) |
| 5877 |
SelectCCOp = PPC::SELECT_CC_SPE4; |
5877 |
SelectCCOp = PPC::SELECT_CC_SPE4; |
| 5878 |
else |
5878 |
else |
| 5879 |
SelectCCOp = PPC::SELECT_CC_F4; |
5879 |
SelectCCOp = PPC::SELECT_CC_F4; |
| 5880 |
} else if (N->getValueType(0) == MVT::f64) { |
5880 |
} else if (N->getValueType(0) == MVT::f64) { |
| 5881 |
if (Subtarget->hasVSX()) |
5881 |
if (Subtarget->hasVSX()) |
| 5882 |
SelectCCOp = PPC::SELECT_CC_VSFRC; |
5882 |
SelectCCOp = PPC::SELECT_CC_VSFRC; |
| 5883 |
else if (Subtarget->hasSPE()) |
5883 |
else if (Subtarget->hasSPE()) |
| 5884 |
SelectCCOp = PPC::SELECT_CC_SPE; |
5884 |
SelectCCOp = PPC::SELECT_CC_SPE; |
| 5885 |
else |
5885 |
else |
| 5886 |
SelectCCOp = PPC::SELECT_CC_F8; |
5886 |
SelectCCOp = PPC::SELECT_CC_F8; |
| 5887 |
} else if (N->getValueType(0) == MVT::f128) |
5887 |
} else if (N->getValueType(0) == MVT::f128) |
| 5888 |
SelectCCOp = PPC::SELECT_CC_F16; |
5888 |
SelectCCOp = PPC::SELECT_CC_F16; |
| 5889 |
else if (Subtarget->hasSPE()) |
5889 |
else if (Subtarget->hasSPE()) |
| 5890 |
SelectCCOp = PPC::SELECT_CC_SPE; |
5890 |
SelectCCOp = PPC::SELECT_CC_SPE; |
| 5891 |
else if (N->getValueType(0) == MVT::v2f64 || |
5891 |
else if (N->getValueType(0) == MVT::v2f64 || |
| 5892 |
N->getValueType(0) == MVT::v2i64) |
5892 |
N->getValueType(0) == MVT::v2i64) |
| 5893 |
SelectCCOp = PPC::SELECT_CC_VSRC; |
5893 |
SelectCCOp = PPC::SELECT_CC_VSRC; |
| 5894 |
else |
5894 |
else |
| 5895 |
SelectCCOp = PPC::SELECT_CC_VRRC; |
5895 |
SelectCCOp = PPC::SELECT_CC_VRRC; |
| 5896 |
|
5896 |
|
| 5897 |
SDValue Ops[] = { CCReg, N->getOperand(2), N->getOperand(3), |
5897 |
SDValue Ops[] = { CCReg, N->getOperand(2), N->getOperand(3), |
| 5898 |
getI32Imm(BROpc, dl) }; |
5898 |
getI32Imm(BROpc, dl) }; |
| 5899 |
CurDAG->SelectNodeTo(N, SelectCCOp, N->getValueType(0), Ops); |
5899 |
CurDAG->SelectNodeTo(N, SelectCCOp, N->getValueType(0), Ops); |
| 5900 |
return; |
5900 |
return; |
| 5901 |
} |
5901 |
} |
| 5902 |
case ISD::VECTOR_SHUFFLE: |
5902 |
case ISD::VECTOR_SHUFFLE: |
| 5903 |
if (Subtarget->hasVSX() && (N->getValueType(0) == MVT::v2f64 || |
5903 |
if (Subtarget->hasVSX() && (N->getValueType(0) == MVT::v2f64 || |
| 5904 |
N->getValueType(0) == MVT::v2i64)) { |
5904 |
N->getValueType(0) == MVT::v2i64)) { |
| 5905 |
ShuffleVectorSDNode *SVN = cast(N); |
5905 |
ShuffleVectorSDNode *SVN = cast(N); |
| 5906 |
|
5906 |
|
| 5907 |
SDValue Op1 = N->getOperand(SVN->getMaskElt(0) < 2 ? 0 : 1), |
5907 |
SDValue Op1 = N->getOperand(SVN->getMaskElt(0) < 2 ? 0 : 1), |
| 5908 |
Op2 = N->getOperand(SVN->getMaskElt(1) < 2 ? 0 : 1); |
5908 |
Op2 = N->getOperand(SVN->getMaskElt(1) < 2 ? 0 : 1); |
| 5909 |
unsigned DM[2]; |
5909 |
unsigned DM[2]; |
| 5910 |
|
5910 |
|
| 5911 |
for (int i = 0; i < 2; ++i) |
5911 |
for (int i = 0; i < 2; ++i) |
| 5912 |
if (SVN->getMaskElt(i) <= 0 || SVN->getMaskElt(i) == 2) |
5912 |
if (SVN->getMaskElt(i) <= 0 || SVN->getMaskElt(i) == 2) |
| 5913 |
DM[i] = 0; |
5913 |
DM[i] = 0; |
| 5914 |
else |
5914 |
else |
| 5915 |
DM[i] = 1; |
5915 |
DM[i] = 1; |
| 5916 |
|
5916 |
|
| 5917 |
if (Op1 == Op2 && DM[0] == 0 && DM[1] == 0 && |
5917 |
if (Op1 == Op2 && DM[0] == 0 && DM[1] == 0 && |
| 5918 |
Op1.getOpcode() == ISD::SCALAR_TO_VECTOR && |
5918 |
Op1.getOpcode() == ISD::SCALAR_TO_VECTOR && |
| 5919 |
isa(Op1.getOperand(0))) { |
5919 |
isa(Op1.getOperand(0))) { |
| 5920 |
LoadSDNode *LD = cast(Op1.getOperand(0)); |
5920 |
LoadSDNode *LD = cast(Op1.getOperand(0)); |
| 5921 |
SDValue Base, Offset; |
5921 |
SDValue Base, Offset; |
| 5922 |
|
5922 |
|
| 5923 |
if (LD->isUnindexed() && LD->hasOneUse() && Op1.hasOneUse() && |
5923 |
if (LD->isUnindexed() && LD->hasOneUse() && Op1.hasOneUse() && |
| 5924 |
(LD->getMemoryVT() == MVT::f64 || |
5924 |
(LD->getMemoryVT() == MVT::f64 || |
| 5925 |
LD->getMemoryVT() == MVT::i64) && |
5925 |
LD->getMemoryVT() == MVT::i64) && |
| 5926 |
SelectAddrIdxOnly(LD->getBasePtr(), Base, Offset)) { |
5926 |
SelectAddrIdxOnly(LD->getBasePtr(), Base, Offset)) { |
| 5927 |
SDValue Chain = LD->getChain(); |
5927 |
SDValue Chain = LD->getChain(); |
| 5928 |
SDValue Ops[] = { Base, Offset, Chain }; |
5928 |
SDValue Ops[] = { Base, Offset, Chain }; |
| 5929 |
MachineMemOperand *MemOp = LD->getMemOperand(); |
5929 |
MachineMemOperand *MemOp = LD->getMemOperand(); |
| 5930 |
SDNode *NewN = CurDAG->SelectNodeTo(N, PPC::LXVDSX, |
5930 |
SDNode *NewN = CurDAG->SelectNodeTo(N, PPC::LXVDSX, |
| 5931 |
N->getValueType(0), Ops); |
5931 |
N->getValueType(0), Ops); |
| 5932 |
CurDAG->setNodeMemRefs(cast(NewN), {MemOp}); |
5932 |
CurDAG->setNodeMemRefs(cast(NewN), {MemOp}); |
| 5933 |
return; |
5933 |
return; |
| 5934 |
} |
5934 |
} |
| 5935 |
} |
5935 |
} |
| 5936 |
|
5936 |
|
| 5937 |
// For little endian, we must swap the input operands and adjust |
5937 |
// For little endian, we must swap the input operands and adjust |
| 5938 |
// the mask elements (reverse and invert them). |
5938 |
// the mask elements (reverse and invert them). |
| 5939 |
if (Subtarget->isLittleEndian()) { |
5939 |
if (Subtarget->isLittleEndian()) { |
| 5940 |
std::swap(Op1, Op2); |
5940 |
std::swap(Op1, Op2); |
| 5941 |
unsigned tmp = DM[0]; |
5941 |
unsigned tmp = DM[0]; |
| 5942 |
DM[0] = 1 - DM[1]; |
5942 |
DM[0] = 1 - DM[1]; |
| 5943 |
DM[1] = 1 - tmp; |
5943 |
DM[1] = 1 - tmp; |
| 5944 |
} |
5944 |
} |
| 5945 |
|
5945 |
|
| 5946 |
SDValue DMV = CurDAG->getTargetConstant(DM[1] | (DM[0] << 1), dl, |
5946 |
SDValue DMV = CurDAG->getTargetConstant(DM[1] | (DM[0] << 1), dl, |
| 5947 |
MVT::i32); |
5947 |
MVT::i32); |
| 5948 |
SDValue Ops[] = { Op1, Op2, DMV }; |
5948 |
SDValue Ops[] = { Op1, Op2, DMV }; |
| 5949 |
CurDAG->SelectNodeTo(N, PPC::XXPERMDI, N->getValueType(0), Ops); |
5949 |
CurDAG->SelectNodeTo(N, PPC::XXPERMDI, N->getValueType(0), Ops); |
| 5950 |
return; |
5950 |
return; |
| 5951 |
} |
5951 |
} |
| 5952 |
|
5952 |
|
| 5953 |
break; |
5953 |
break; |
| 5954 |
case PPCISD::BDNZ: |
5954 |
case PPCISD::BDNZ: |
| 5955 |
case PPCISD::BDZ: { |
5955 |
case PPCISD::BDZ: { |
| 5956 |
bool IsPPC64 = Subtarget->isPPC64(); |
5956 |
bool IsPPC64 = Subtarget->isPPC64(); |
| 5957 |
SDValue Ops[] = { N->getOperand(1), N->getOperand(0) }; |
5957 |
SDValue Ops[] = { N->getOperand(1), N->getOperand(0) }; |
| 5958 |
CurDAG->SelectNodeTo(N, N->getOpcode() == PPCISD::BDNZ |
5958 |
CurDAG->SelectNodeTo(N, N->getOpcode() == PPCISD::BDNZ |
| 5959 |
? (IsPPC64 ? PPC::BDNZ8 : PPC::BDNZ) |
5959 |
? (IsPPC64 ? PPC::BDNZ8 : PPC::BDNZ) |
| 5960 |
: (IsPPC64 ? PPC::BDZ8 : PPC::BDZ), |
5960 |
: (IsPPC64 ? PPC::BDZ8 : PPC::BDZ), |
| 5961 |
MVT::Other, Ops); |
5961 |
MVT::Other, Ops); |
| 5962 |
return; |
5962 |
return; |
| 5963 |
} |
5963 |
} |
| 5964 |
case PPCISD::COND_BRANCH: { |
5964 |
case PPCISD::COND_BRANCH: { |
| 5965 |
// Op #0 is the Chain. |
5965 |
// Op #0 is the Chain. |
| 5966 |
// Op #1 is the PPC::PRED_* number. |
5966 |
// Op #1 is the PPC::PRED_* number. |
| 5967 |
// Op #2 is the CR# |
5967 |
// Op #2 is the CR# |
| 5968 |
// Op #3 is the Dest MBB |
5968 |
// Op #3 is the Dest MBB |
| 5969 |
// Op #4 is the Flag. |
5969 |
// Op #4 is the Flag. |
| 5970 |
// Prevent PPC::PRED_* from being selected into LI. |
5970 |
// Prevent PPC::PRED_* from being selected into LI. |
| 5971 |
unsigned PCC = cast(N->getOperand(1))->getZExtValue(); |
5971 |
unsigned PCC = cast(N->getOperand(1))->getZExtValue(); |
| 5972 |
if (EnableBranchHint) |
5972 |
if (EnableBranchHint) |
| 5973 |
PCC |= getBranchHint(PCC, *FuncInfo, N->getOperand(3)); |
5973 |
PCC |= getBranchHint(PCC, *FuncInfo, N->getOperand(3)); |
| 5974 |
|
5974 |
|
| 5975 |
SDValue Pred = getI32Imm(PCC, dl); |
5975 |
SDValue Pred = getI32Imm(PCC, dl); |
| 5976 |
SDValue Ops[] = { Pred, N->getOperand(2), N->getOperand(3), |
5976 |
SDValue Ops[] = { Pred, N->getOperand(2), N->getOperand(3), |
| 5977 |
N->getOperand(0), N->getOperand(4) }; |
5977 |
N->getOperand(0), N->getOperand(4) }; |
| 5978 |
CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); |
5978 |
CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); |
| 5979 |
return; |
5979 |
return; |
| 5980 |
} |
5980 |
} |
| 5981 |
case ISD::BR_CC: { |
5981 |
case ISD::BR_CC: { |
| 5982 |
if (tryFoldSWTestBRCC(N)) |
5982 |
if (tryFoldSWTestBRCC(N)) |
| 5983 |
return; |
5983 |
return; |
| 5984 |
if (trySelectLoopCountIntrinsic(N)) |
5984 |
if (trySelectLoopCountIntrinsic(N)) |
| 5985 |
return; |
5985 |
return; |
| 5986 |
ISD::CondCode CC = cast(N->getOperand(1))->get(); |
5986 |
ISD::CondCode CC = cast(N->getOperand(1))->get(); |
| 5987 |
unsigned PCC = |
5987 |
unsigned PCC = |
| 5988 |
getPredicateForSetCC(CC, N->getOperand(2).getValueType(), Subtarget); |
5988 |
getPredicateForSetCC(CC, N->getOperand(2).getValueType(), Subtarget); |
| 5989 |
|
5989 |
|
| 5990 |
if (N->getOperand(2).getValueType() == MVT::i1) { |
5990 |
if (N->getOperand(2).getValueType() == MVT::i1) { |
| 5991 |
unsigned Opc; |
5991 |
unsigned Opc; |
| 5992 |
bool Swap; |
5992 |
bool Swap; |
| 5993 |
switch (PCC) { |
5993 |
switch (PCC) { |
| 5994 |
default: llvm_unreachable("Unexpected Boolean-operand predicate"); |
5994 |
default: llvm_unreachable("Unexpected Boolean-operand predicate"); |
| 5995 |
case PPC::PRED_LT: Opc = PPC::CRANDC; Swap = true; break; |
5995 |
case PPC::PRED_LT: Opc = PPC::CRANDC; Swap = true; break; |
| 5996 |
case PPC::PRED_LE: Opc = PPC::CRORC; Swap = true; break; |
5996 |
case PPC::PRED_LE: Opc = PPC::CRORC; Swap = true; break; |
| 5997 |
case PPC::PRED_EQ: Opc = PPC::CREQV; Swap = false; break; |
5997 |
case PPC::PRED_EQ: Opc = PPC::CREQV; Swap = false; break; |
| 5998 |
case PPC::PRED_GE: Opc = PPC::CRORC; Swap = false; break; |
5998 |
case PPC::PRED_GE: Opc = PPC::CRORC; Swap = false; break; |
| 5999 |
case PPC::PRED_GT: Opc = PPC::CRANDC; Swap = false; break; |
5999 |
case PPC::PRED_GT: Opc = PPC::CRANDC; Swap = false; break; |
| 6000 |
case PPC::PRED_NE: Opc = PPC::CRXOR; Swap = false; break; |
6000 |
case PPC::PRED_NE: Opc = PPC::CRXOR; Swap = false; break; |
| 6001 |
} |
6001 |
} |
| 6002 |
|
6002 |
|
| 6003 |
// A signed comparison of i1 values produces the opposite result to an |
6003 |
// A signed comparison of i1 values produces the opposite result to an |
| 6004 |
// unsigned one if the condition code includes less-than or greater-than. |
6004 |
// unsigned one if the condition code includes less-than or greater-than. |
| 6005 |
// This is because 1 is the most negative signed i1 number and the most |
6005 |
// This is because 1 is the most negative signed i1 number and the most |
| 6006 |
// positive unsigned i1 number. The CR-logical operations used for such |
6006 |
// positive unsigned i1 number. The CR-logical operations used for such |
| 6007 |
// comparisons are non-commutative so for signed comparisons vs. unsigned |
6007 |
// comparisons are non-commutative so for signed comparisons vs. unsigned |
| 6008 |
// ones, the input operands just need to be swapped. |
6008 |
// ones, the input operands just need to be swapped. |
| 6009 |
if (ISD::isSignedIntSetCC(CC)) |
6009 |
if (ISD::isSignedIntSetCC(CC)) |
| 6010 |
Swap = !Swap; |
6010 |
Swap = !Swap; |
| 6011 |
|
6011 |
|
| 6012 |
SDValue BitComp(CurDAG->getMachineNode(Opc, dl, MVT::i1, |
6012 |
SDValue BitComp(CurDAG->getMachineNode(Opc, dl, MVT::i1, |
| 6013 |
N->getOperand(Swap ? 3 : 2), |
6013 |
N->getOperand(Swap ? 3 : 2), |
| 6014 |
N->getOperand(Swap ? 2 : 3)), 0); |
6014 |
N->getOperand(Swap ? 2 : 3)), 0); |
| 6015 |
CurDAG->SelectNodeTo(N, PPC::BC, MVT::Other, BitComp, N->getOperand(4), |
6015 |
CurDAG->SelectNodeTo(N, PPC::BC, MVT::Other, BitComp, N->getOperand(4), |
| 6016 |
N->getOperand(0)); |
6016 |
N->getOperand(0)); |
| 6017 |
return; |
6017 |
return; |
| 6018 |
} |
6018 |
} |
| 6019 |
|
6019 |
|
| 6020 |
if (EnableBranchHint) |
6020 |
if (EnableBranchHint) |
| 6021 |
PCC |= getBranchHint(PCC, *FuncInfo, N->getOperand(4)); |
6021 |
PCC |= getBranchHint(PCC, *FuncInfo, N->getOperand(4)); |
| 6022 |
|
6022 |
|
| 6023 |
SDValue CondCode = SelectCC(N->getOperand(2), N->getOperand(3), CC, dl); |
6023 |
SDValue CondCode = SelectCC(N->getOperand(2), N->getOperand(3), CC, dl); |
| 6024 |
SDValue Ops[] = { getI32Imm(PCC, dl), CondCode, |
6024 |
SDValue Ops[] = { getI32Imm(PCC, dl), CondCode, |
| 6025 |
N->getOperand(4), N->getOperand(0) }; |
6025 |
N->getOperand(4), N->getOperand(0) }; |
| 6026 |
CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); |
6026 |
CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); |
| 6027 |
return; |
6027 |
return; |
| 6028 |
} |
6028 |
} |
| 6029 |
case ISD::BRIND: { |
6029 |
case ISD::BRIND: { |
| 6030 |
// FIXME: Should custom lower this. |
6030 |
// FIXME: Should custom lower this. |
| 6031 |
SDValue Chain = N->getOperand(0); |
6031 |
SDValue Chain = N->getOperand(0); |
| 6032 |
SDValue Target = N->getOperand(1); |
6032 |
SDValue Target = N->getOperand(1); |
| 6033 |
unsigned Opc = Target.getValueType() == MVT::i32 ? PPC::MTCTR : PPC::MTCTR8; |
6033 |
unsigned Opc = Target.getValueType() == MVT::i32 ? PPC::MTCTR : PPC::MTCTR8; |
| 6034 |
unsigned Reg = Target.getValueType() == MVT::i32 ? PPC::BCTR : PPC::BCTR8; |
6034 |
unsigned Reg = Target.getValueType() == MVT::i32 ? PPC::BCTR : PPC::BCTR8; |
| 6035 |
Chain = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, Target, |
6035 |
Chain = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, Target, |
| 6036 |
Chain), 0); |
6036 |
Chain), 0); |
| 6037 |
CurDAG->SelectNodeTo(N, Reg, MVT::Other, Chain); |
6037 |
CurDAG->SelectNodeTo(N, Reg, MVT::Other, Chain); |
| 6038 |
return; |
6038 |
return; |
| 6039 |
} |
6039 |
} |
| 6040 |
case PPCISD::TOC_ENTRY: { |
6040 |
case PPCISD::TOC_ENTRY: { |
| 6041 |
const bool isPPC64 = Subtarget->isPPC64(); |
6041 |
const bool isPPC64 = Subtarget->isPPC64(); |
| 6042 |
const bool isELFABI = Subtarget->isSVR4ABI(); |
6042 |
const bool isELFABI = Subtarget->isSVR4ABI(); |
| 6043 |
const bool isAIXABI = Subtarget->isAIXABI(); |
6043 |
const bool isAIXABI = Subtarget->isAIXABI(); |
| 6044 |
|
6044 |
|
| 6045 |
// PowerPC only support small, medium and large code model. |
6045 |
// PowerPC only support small, medium and large code model. |
| 6046 |
const CodeModel::Model CModel = TM.getCodeModel(); |
6046 |
const CodeModel::Model CModel = TM.getCodeModel(); |
| 6047 |
assert(!(CModel == CodeModel::Tiny || CModel == CodeModel::Kernel) && |
6047 |
assert(!(CModel == CodeModel::Tiny || CModel == CodeModel::Kernel) && |
| 6048 |
"PowerPC doesn't support tiny or kernel code models."); |
6048 |
"PowerPC doesn't support tiny or kernel code models."); |
| 6049 |
|
6049 |
|
| 6050 |
if (isAIXABI && CModel == CodeModel::Medium) |
6050 |
if (isAIXABI && CModel == CodeModel::Medium) |
| 6051 |
report_fatal_error("Medium code model is not supported on AIX."); |
6051 |
report_fatal_error("Medium code model is not supported on AIX."); |
| 6052 |
|
6052 |
|
| 6053 |
// For 64-bit ELF small code model, we allow SelectCodeCommon to handle |
6053 |
// For 64-bit ELF small code model, we allow SelectCodeCommon to handle |
| 6054 |
// this, selecting one of LDtoc, LDtocJTI, LDtocCPT, and LDtocBA. For AIX |
6054 |
// this, selecting one of LDtoc, LDtocJTI, LDtocCPT, and LDtocBA. For AIX |
| 6055 |
// small code model, we need to check for a toc-data attribute. |
6055 |
// small code model, we need to check for a toc-data attribute. |
| 6056 |
if (isPPC64 && !isAIXABI && CModel == CodeModel::Small) |
6056 |
if (isPPC64 && !isAIXABI && CModel == CodeModel::Small) |
| 6057 |
break; |
6057 |
break; |
| 6058 |
|
6058 |
|
| 6059 |
auto replaceWith = [this, &dl](unsigned OpCode, SDNode *TocEntry, |
6059 |
auto replaceWith = [this, &dl](unsigned OpCode, SDNode *TocEntry, |
| 6060 |
EVT OperandTy) { |
6060 |
EVT OperandTy) { |
| 6061 |
SDValue GA = TocEntry->getOperand(0); |
6061 |
SDValue GA = TocEntry->getOperand(0); |
| 6062 |
SDValue TocBase = TocEntry->getOperand(1); |
6062 |
SDValue TocBase = TocEntry->getOperand(1); |
| 6063 |
SDNode *MN = CurDAG->getMachineNode(OpCode, dl, OperandTy, GA, TocBase); |
6063 |
SDNode *MN = CurDAG->getMachineNode(OpCode, dl, OperandTy, GA, TocBase); |
| 6064 |
transferMemOperands(TocEntry, MN); |
6064 |
transferMemOperands(TocEntry, MN); |
| 6065 |
ReplaceNode(TocEntry, MN); |
6065 |
ReplaceNode(TocEntry, MN); |
| 6066 |
}; |
6066 |
}; |
| 6067 |
|
6067 |
|
| 6068 |
// Handle 32-bit small code model. |
6068 |
// Handle 32-bit small code model. |
| 6069 |
if (!isPPC64 && CModel == CodeModel::Small) { |
6069 |
if (!isPPC64 && CModel == CodeModel::Small) { |
| 6070 |
// Transforms the ISD::TOC_ENTRY node to passed in Opcode, either |
6070 |
// Transforms the ISD::TOC_ENTRY node to passed in Opcode, either |
| 6071 |
// PPC::ADDItoc, or PPC::LWZtoc |
6071 |
// PPC::ADDItoc, or PPC::LWZtoc |
| 6072 |
if (isELFABI) { |
6072 |
if (isELFABI) { |
| 6073 |
assert(TM.isPositionIndependent() && |
6073 |
assert(TM.isPositionIndependent() && |
| 6074 |
"32-bit ELF can only have TOC entries in position independent" |
6074 |
"32-bit ELF can only have TOC entries in position independent" |
| 6075 |
" code."); |
6075 |
" code."); |
| 6076 |
// 32-bit ELF always uses a small code model toc access. |
6076 |
// 32-bit ELF always uses a small code model toc access. |
| 6077 |
replaceWith(PPC::LWZtoc, N, MVT::i32); |
6077 |
replaceWith(PPC::LWZtoc, N, MVT::i32); |
| 6078 |
return; |
6078 |
return; |
| 6079 |
} |
6079 |
} |
| 6080 |
|
6080 |
|
| 6081 |
assert(isAIXABI && "ELF ABI already handled"); |
6081 |
assert(isAIXABI && "ELF ABI already handled"); |
| 6082 |
|
6082 |
|
| 6083 |
if (hasTocDataAttr(N->getOperand(0), |
6083 |
if (hasTocDataAttr(N->getOperand(0), |
| 6084 |
CurDAG->getDataLayout().getPointerSize())) { |
6084 |
CurDAG->getDataLayout().getPointerSize())) { |
| 6085 |
replaceWith(PPC::ADDItoc, N, MVT::i32); |
6085 |
replaceWith(PPC::ADDItoc, N, MVT::i32); |
| 6086 |
return; |
6086 |
return; |
| 6087 |
} |
6087 |
} |
| 6088 |
|
6088 |
|
| 6089 |
replaceWith(PPC::LWZtoc, N, MVT::i32); |
6089 |
replaceWith(PPC::LWZtoc, N, MVT::i32); |
| 6090 |
return; |
6090 |
return; |
| 6091 |
} |
6091 |
} |
| 6092 |
|
6092 |
|
| 6093 |
if (isPPC64 && CModel == CodeModel::Small) { |
6093 |
if (isPPC64 && CModel == CodeModel::Small) { |
| 6094 |
assert(isAIXABI && "ELF ABI handled in common SelectCode"); |
6094 |
assert(isAIXABI && "ELF ABI handled in common SelectCode"); |
| 6095 |
|
6095 |
|
| 6096 |
if (hasTocDataAttr(N->getOperand(0), |
6096 |
if (hasTocDataAttr(N->getOperand(0), |
| 6097 |
CurDAG->getDataLayout().getPointerSize())) { |
6097 |
CurDAG->getDataLayout().getPointerSize())) { |
| 6098 |
replaceWith(PPC::ADDItoc8, N, MVT::i64); |
6098 |
replaceWith(PPC::ADDItoc8, N, MVT::i64); |
| 6099 |
return; |
6099 |
return; |
| 6100 |
} |
6100 |
} |
| 6101 |
// Break if it doesn't have toc data attribute. Proceed with common |
6101 |
// Break if it doesn't have toc data attribute. Proceed with common |
| 6102 |
// SelectCode. |
6102 |
// SelectCode. |
| 6103 |
break; |
6103 |
break; |
| 6104 |
} |
6104 |
} |
| 6105 |
|
6105 |
|
| 6106 |
assert(CModel != CodeModel::Small && "All small code models handled."); |
6106 |
assert(CModel != CodeModel::Small && "All small code models handled."); |
| 6107 |
|
6107 |
|
| 6108 |
assert((isPPC64 || (isAIXABI && !isPPC64)) && "We are dealing with 64-bit" |
6108 |
assert((isPPC64 || (isAIXABI && !isPPC64)) && "We are dealing with 64-bit" |
| 6109 |
" ELF/AIX or 32-bit AIX in the following."); |
6109 |
" ELF/AIX or 32-bit AIX in the following."); |
| 6110 |
|
6110 |
|
| 6111 |
// Transforms the ISD::TOC_ENTRY node for 32-bit AIX large code model mode |
6111 |
// Transforms the ISD::TOC_ENTRY node for 32-bit AIX large code model mode |
| 6112 |
// or 64-bit medium (ELF-only) or large (ELF and AIX) code model code. We |
6112 |
// or 64-bit medium (ELF-only) or large (ELF and AIX) code model code. We |
| 6113 |
// generate two instructions as described below. The first source operand |
6113 |
// generate two instructions as described below. The first source operand |
| 6114 |
// is a symbol reference. If it must be toc-referenced according to |
6114 |
// is a symbol reference. If it must be toc-referenced according to |
| 6115 |
// Subtarget, we generate: |
6115 |
// Subtarget, we generate: |
| 6116 |
// [32-bit AIX] |
6116 |
// [32-bit AIX] |
| 6117 |
// LWZtocL(@sym, ADDIStocHA(%r2, @sym)) |
6117 |
// LWZtocL(@sym, ADDIStocHA(%r2, @sym)) |
| 6118 |
// [64-bit ELF/AIX] |
6118 |
// [64-bit ELF/AIX] |
| 6119 |
// LDtocL(@sym, ADDIStocHA8(%x2, @sym)) |
6119 |
// LDtocL(@sym, ADDIStocHA8(%x2, @sym)) |
| 6120 |
// Otherwise we generate: |
6120 |
// Otherwise we generate: |
| 6121 |
// ADDItocL(ADDIStocHA8(%x2, @sym), @sym) |
6121 |
// ADDItocL(ADDIStocHA8(%x2, @sym), @sym) |
| 6122 |
SDValue GA = N->getOperand(0); |
6122 |
SDValue GA = N->getOperand(0); |
| 6123 |
SDValue TOCbase = N->getOperand(1); |
6123 |
SDValue TOCbase = N->getOperand(1); |
| 6124 |
|
6124 |
|
| 6125 |
EVT VT = isPPC64 ? MVT::i64 : MVT::i32; |
6125 |
EVT VT = isPPC64 ? MVT::i64 : MVT::i32; |
| 6126 |
SDNode *Tmp = CurDAG->getMachineNode( |
6126 |
SDNode *Tmp = CurDAG->getMachineNode( |
| 6127 |
isPPC64 ? PPC::ADDIStocHA8 : PPC::ADDIStocHA, dl, VT, TOCbase, GA); |
6127 |
isPPC64 ? PPC::ADDIStocHA8 : PPC::ADDIStocHA, dl, VT, TOCbase, GA); |
| 6128 |
|
6128 |
|
| 6129 |
if (PPCLowering->isAccessedAsGotIndirect(GA)) { |
6129 |
if (PPCLowering->isAccessedAsGotIndirect(GA)) { |
| 6130 |
// If it is accessed as got-indirect, we need an extra LWZ/LD to load |
6130 |
// If it is accessed as got-indirect, we need an extra LWZ/LD to load |
| 6131 |
// the address. |
6131 |
// the address. |
| 6132 |
SDNode *MN = CurDAG->getMachineNode( |
6132 |
SDNode *MN = CurDAG->getMachineNode( |
| 6133 |
isPPC64 ? PPC::LDtocL : PPC::LWZtocL, dl, VT, GA, SDValue(Tmp, 0)); |
6133 |
isPPC64 ? PPC::LDtocL : PPC::LWZtocL, dl, VT, GA, SDValue(Tmp, 0)); |
| 6134 |
|
6134 |
|
| 6135 |
transferMemOperands(N, MN); |
6135 |
transferMemOperands(N, MN); |
| 6136 |
ReplaceNode(N, MN); |
6136 |
ReplaceNode(N, MN); |
| 6137 |
return; |
6137 |
return; |
| 6138 |
} |
6138 |
} |
| 6139 |
|
6139 |
|
| 6140 |
// Build the address relative to the TOC-pointer. |
6140 |
// Build the address relative to the TOC-pointer. |
| 6141 |
ReplaceNode(N, CurDAG->getMachineNode(PPC::ADDItocL, dl, MVT::i64, |
6141 |
ReplaceNode(N, CurDAG->getMachineNode(PPC::ADDItocL, dl, MVT::i64, |
| 6142 |
SDValue(Tmp, 0), GA)); |
6142 |
SDValue(Tmp, 0), GA)); |
| 6143 |
return; |
6143 |
return; |
| 6144 |
} |
6144 |
} |
| 6145 |
case PPCISD::PPC32_PICGOT: |
6145 |
case PPCISD::PPC32_PICGOT: |
| 6146 |
// Generate a PIC-safe GOT reference. |
6146 |
// Generate a PIC-safe GOT reference. |
| 6147 |
assert(Subtarget->is32BitELFABI() && |
6147 |
assert(Subtarget->is32BitELFABI() && |
| 6148 |
"PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4"); |
6148 |
"PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4"); |
| 6149 |
CurDAG->SelectNodeTo(N, PPC::PPC32PICGOT, |
6149 |
CurDAG->SelectNodeTo(N, PPC::PPC32PICGOT, |
| 6150 |
PPCLowering->getPointerTy(CurDAG->getDataLayout()), |
6150 |
PPCLowering->getPointerTy(CurDAG->getDataLayout()), |
| 6151 |
MVT::i32); |
6151 |
MVT::i32); |
| 6152 |
return; |
6152 |
return; |
| 6153 |
|
6153 |
|
| 6154 |
case PPCISD::VADD_SPLAT: { |
6154 |
case PPCISD::VADD_SPLAT: { |
| 6155 |
// This expands into one of three sequences, depending on whether |
6155 |
// This expands into one of three sequences, depending on whether |
| 6156 |
// the first operand is odd or even, positive or negative. |
6156 |
// the first operand is odd or even, positive or negative. |
| 6157 |
assert(isa(N->getOperand(0)) && |
6157 |
assert(isa(N->getOperand(0)) && |
| 6158 |
isa(N->getOperand(1)) && |
6158 |
isa(N->getOperand(1)) && |
| 6159 |
"Invalid operand on VADD_SPLAT!"); |
6159 |
"Invalid operand on VADD_SPLAT!"); |
| 6160 |
|
6160 |
|
| 6161 |
int Elt = N->getConstantOperandVal(0); |
6161 |
int Elt = N->getConstantOperandVal(0); |
| 6162 |
int EltSize = N->getConstantOperandVal(1); |
6162 |
int EltSize = N->getConstantOperandVal(1); |
| 6163 |
unsigned Opc1, Opc2, Opc3; |
6163 |
unsigned Opc1, Opc2, Opc3; |
| 6164 |
EVT VT; |
6164 |
EVT VT; |
| 6165 |
|
6165 |
|
| 6166 |
if (EltSize == 1) { |
6166 |
if (EltSize == 1) { |
| 6167 |
Opc1 = PPC::VSPLTISB; |
6167 |
Opc1 = PPC::VSPLTISB; |
| 6168 |
Opc2 = PPC::VADDUBM; |
6168 |
Opc2 = PPC::VADDUBM; |
| 6169 |
Opc3 = PPC::VSUBUBM; |
6169 |
Opc3 = PPC::VSUBUBM; |
| 6170 |
VT = MVT::v16i8; |
6170 |
VT = MVT::v16i8; |
| 6171 |
} else if (EltSize == 2) { |
6171 |
} else if (EltSize == 2) { |
| 6172 |
Opc1 = PPC::VSPLTISH; |
6172 |
Opc1 = PPC::VSPLTISH; |
| 6173 |
Opc2 = PPC::VADDUHM; |
6173 |
Opc2 = PPC::VADDUHM; |
| 6174 |
Opc3 = PPC::VSUBUHM; |
6174 |
Opc3 = PPC::VSUBUHM; |
| 6175 |
VT = MVT::v8i16; |
6175 |
VT = MVT::v8i16; |
| 6176 |
} else { |
6176 |
} else { |
| 6177 |
assert(EltSize == 4 && "Invalid element size on VADD_SPLAT!"); |
6177 |
assert(EltSize == 4 && "Invalid element size on VADD_SPLAT!"); |
| 6178 |
Opc1 = PPC::VSPLTISW; |
6178 |
Opc1 = PPC::VSPLTISW; |
| 6179 |
Opc2 = PPC::VADDUWM; |
6179 |
Opc2 = PPC::VADDUWM; |
| 6180 |
Opc3 = PPC::VSUBUWM; |
6180 |
Opc3 = PPC::VSUBUWM; |
| 6181 |
VT = MVT::v4i32; |
6181 |
VT = MVT::v4i32; |
| 6182 |
} |
6182 |
} |
| 6183 |
|
6183 |
|
| 6184 |
if ((Elt & 1) == 0) { |
6184 |
if ((Elt & 1) == 0) { |
| 6185 |
// Elt is even, in the range [-32,-18] + [16,30]. |
6185 |
// Elt is even, in the range [-32,-18] + [16,30]. |
| 6186 |
// |
6186 |
// |
| 6187 |
// Convert: VADD_SPLAT elt, size |
6187 |
// Convert: VADD_SPLAT elt, size |
| 6188 |
// Into: tmp = VSPLTIS[BHW] elt |
6188 |
// Into: tmp = VSPLTIS[BHW] elt |
| 6189 |
// VADDU[BHW]M tmp, tmp |
6189 |
// VADDU[BHW]M tmp, tmp |
| 6190 |
// Where: [BHW] = B for size = 1, H for size = 2, W for size = 4 |
6190 |
// Where: [BHW] = B for size = 1, H for size = 2, W for size = 4 |
| 6191 |
SDValue EltVal = getI32Imm(Elt >> 1, dl); |
6191 |
SDValue EltVal = getI32Imm(Elt >> 1, dl); |
| 6192 |
SDNode *Tmp = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); |
6192 |
SDNode *Tmp = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); |
| 6193 |
SDValue TmpVal = SDValue(Tmp, 0); |
6193 |
SDValue TmpVal = SDValue(Tmp, 0); |
| 6194 |
ReplaceNode(N, CurDAG->getMachineNode(Opc2, dl, VT, TmpVal, TmpVal)); |
6194 |
ReplaceNode(N, CurDAG->getMachineNode(Opc2, dl, VT, TmpVal, TmpVal)); |
| 6195 |
return; |
6195 |
return; |
| 6196 |
} else if (Elt > 0) { |
6196 |
} else if (Elt > 0) { |
| 6197 |
// Elt is odd and positive, in the range [17,31]. |
6197 |
// Elt is odd and positive, in the range [17,31]. |
| 6198 |
// |
6198 |
// |
| 6199 |
// Convert: VADD_SPLAT elt, size |
6199 |
// Convert: VADD_SPLAT elt, size |
| 6200 |
// Into: tmp1 = VSPLTIS[BHW] elt-16 |
6200 |
// Into: tmp1 = VSPLTIS[BHW] elt-16 |
| 6201 |
// tmp2 = VSPLTIS[BHW] -16 |
6201 |
// tmp2 = VSPLTIS[BHW] -16 |
| 6202 |
// VSUBU[BHW]M tmp1, tmp2 |
6202 |
// VSUBU[BHW]M tmp1, tmp2 |
| 6203 |
SDValue EltVal = getI32Imm(Elt - 16, dl); |
6203 |
SDValue EltVal = getI32Imm(Elt - 16, dl); |
| 6204 |
SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); |
6204 |
SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); |
| 6205 |
EltVal = getI32Imm(-16, dl); |
6205 |
EltVal = getI32Imm(-16, dl); |
| 6206 |
SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); |
6206 |
SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); |
| 6207 |
ReplaceNode(N, CurDAG->getMachineNode(Opc3, dl, VT, SDValue(Tmp1, 0), |
6207 |
ReplaceNode(N, CurDAG->getMachineNode(Opc3, dl, VT, SDValue(Tmp1, 0), |
| 6208 |
SDValue(Tmp2, 0))); |
6208 |
SDValue(Tmp2, 0))); |
| 6209 |
return; |
6209 |
return; |
| 6210 |
} else { |
6210 |
} else { |
| 6211 |
// Elt is odd and negative, in the range [-31,-17]. |
6211 |
// Elt is odd and negative, in the range [-31,-17]. |
| 6212 |
// |
6212 |
// |
| 6213 |
// Convert: VADD_SPLAT elt, size |
6213 |
// Convert: VADD_SPLAT elt, size |
| 6214 |
// Into: tmp1 = VSPLTIS[BHW] elt+16 |
6214 |
// Into: tmp1 = VSPLTIS[BHW] elt+16 |
| 6215 |
// tmp2 = VSPLTIS[BHW] -16 |
6215 |
// tmp2 = VSPLTIS[BHW] -16 |
| 6216 |
// VADDU[BHW]M tmp1, tmp2 |
6216 |
// VADDU[BHW]M tmp1, tmp2 |
| 6217 |
SDValue EltVal = getI32Imm(Elt + 16, dl); |
6217 |
SDValue EltVal = getI32Imm(Elt + 16, dl); |
| 6218 |
SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); |
6218 |
SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); |
| 6219 |
EltVal = getI32Imm(-16, dl); |
6219 |
EltVal = getI32Imm(-16, dl); |
| 6220 |
SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); |
6220 |
SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); |
| 6221 |
ReplaceNode(N, CurDAG->getMachineNode(Opc2, dl, VT, SDValue(Tmp1, 0), |
6221 |
ReplaceNode(N, CurDAG->getMachineNode(Opc2, dl, VT, SDValue(Tmp1, 0), |
| 6222 |
SDValue(Tmp2, 0))); |
6222 |
SDValue(Tmp2, 0))); |
| 6223 |
return; |
6223 |
return; |
| 6224 |
} |
6224 |
} |
| 6225 |
} |
6225 |
} |
| 6226 |
case PPCISD::LD_SPLAT: { |
6226 |
case PPCISD::LD_SPLAT: { |
| 6227 |
// Here we want to handle splat load for type v16i8 and v8i16 when there is |
6227 |
// Here we want to handle splat load for type v16i8 and v8i16 when there is |
| 6228 |
// no direct move, we don't need to use stack for this case. If target has |
6228 |
// no direct move, we don't need to use stack for this case. If target has |
| 6229 |
// direct move, we should be able to get the best selection in the .td file. |
6229 |
// direct move, we should be able to get the best selection in the .td file. |
| 6230 |
if (!Subtarget->hasAltivec() || Subtarget->hasDirectMove()) |
6230 |
if (!Subtarget->hasAltivec() || Subtarget->hasDirectMove()) |
| 6231 |
break; |
6231 |
break; |
| 6232 |
|
6232 |
|
| 6233 |
EVT Type = N->getValueType(0); |
6233 |
EVT Type = N->getValueType(0); |
| 6234 |
if (Type != MVT::v16i8 && Type != MVT::v8i16) |
6234 |
if (Type != MVT::v16i8 && Type != MVT::v8i16) |
| 6235 |
break; |
6235 |
break; |
| 6236 |
|
6236 |
|
| 6237 |
// If the alignment for the load is 16 or bigger, we don't need the |
6237 |
// If the alignment for the load is 16 or bigger, we don't need the |
| 6238 |
// permutated mask to get the required value. The value must be the 0 |
6238 |
// permutated mask to get the required value. The value must be the 0 |
| 6239 |
// element in big endian target or 7/15 in little endian target in the |
6239 |
// element in big endian target or 7/15 in little endian target in the |
| 6240 |
// result vsx register of lvx instruction. |
6240 |
// result vsx register of lvx instruction. |
| 6241 |
// Select the instruction in the .td file. |
6241 |
// Select the instruction in the .td file. |
| 6242 |
if (cast(N)->getAlign() >= Align(16) && |
6242 |
if (cast(N)->getAlign() >= Align(16) && |
| 6243 |
isOffsetMultipleOf(N, 16)) |
6243 |
isOffsetMultipleOf(N, 16)) |
| 6244 |
break; |
6244 |
break; |
| 6245 |
|
6245 |
|
| 6246 |
SDValue ZeroReg = |
6246 |
SDValue ZeroReg = |
| 6247 |
CurDAG->getRegister(Subtarget->isPPC64() ? PPC::ZERO8 : PPC::ZERO, |
6247 |
CurDAG->getRegister(Subtarget->isPPC64() ? PPC::ZERO8 : PPC::ZERO, |
| 6248 |
Subtarget->isPPC64() ? MVT::i64 : MVT::i32); |
6248 |
Subtarget->isPPC64() ? MVT::i64 : MVT::i32); |
| 6249 |
unsigned LIOpcode = Subtarget->isPPC64() ? PPC::LI8 : PPC::LI; |
6249 |
unsigned LIOpcode = Subtarget->isPPC64() ? PPC::LI8 : PPC::LI; |
| 6250 |
// v16i8 LD_SPLAT addr |
6250 |
// v16i8 LD_SPLAT addr |
| 6251 |
// ======> |
6251 |
// ======> |
| 6252 |
// Mask = LVSR/LVSL 0, addr |
6252 |
// Mask = LVSR/LVSL 0, addr |
| 6253 |
// LoadLow = LVX 0, addr |
6253 |
// LoadLow = LVX 0, addr |
| 6254 |
// Perm = VPERM LoadLow, LoadLow, Mask |
6254 |
// Perm = VPERM LoadLow, LoadLow, Mask |
| 6255 |
// Splat = VSPLTB 15/0, Perm |
6255 |
// Splat = VSPLTB 15/0, Perm |
| 6256 |
// |
6256 |
// |
| 6257 |
// v8i16 LD_SPLAT addr |
6257 |
// v8i16 LD_SPLAT addr |
| 6258 |
// ======> |
6258 |
// ======> |
| 6259 |
// Mask = LVSR/LVSL 0, addr |
6259 |
// Mask = LVSR/LVSL 0, addr |
| 6260 |
// LoadLow = LVX 0, addr |
6260 |
// LoadLow = LVX 0, addr |
| 6261 |
// LoadHigh = LVX (LI, 1), addr |
6261 |
// LoadHigh = LVX (LI, 1), addr |
| 6262 |
// Perm = VPERM LoadLow, LoadHigh, Mask |
6262 |
// Perm = VPERM LoadLow, LoadHigh, Mask |
| 6263 |
// Splat = VSPLTH 7/0, Perm |
6263 |
// Splat = VSPLTH 7/0, Perm |
| 6264 |
unsigned SplatOp = (Type == MVT::v16i8) ? PPC::VSPLTB : PPC::VSPLTH; |
6264 |
unsigned SplatOp = (Type == MVT::v16i8) ? PPC::VSPLTB : PPC::VSPLTH; |
| 6265 |
unsigned SplatElemIndex = |
6265 |
unsigned SplatElemIndex = |
| 6266 |
Subtarget->isLittleEndian() ? ((Type == MVT::v16i8) ? 15 : 7) : 0; |
6266 |
Subtarget->isLittleEndian() ? ((Type == MVT::v16i8) ? 15 : 7) : 0; |
| 6267 |
|
6267 |
|
| 6268 |
SDNode *Mask = CurDAG->getMachineNode( |
6268 |
SDNode *Mask = CurDAG->getMachineNode( |
| 6269 |
Subtarget->isLittleEndian() ? PPC::LVSR : PPC::LVSL, dl, Type, ZeroReg, |
6269 |
Subtarget->isLittleEndian() ? PPC::LVSR : PPC::LVSL, dl, Type, ZeroReg, |
| 6270 |
N->getOperand(1)); |
6270 |
N->getOperand(1)); |
| 6271 |
|
6271 |
|
| 6272 |
SDNode *LoadLow = |
6272 |
SDNode *LoadLow = |
| 6273 |
CurDAG->getMachineNode(PPC::LVX, dl, MVT::v16i8, MVT::Other, |
6273 |
CurDAG->getMachineNode(PPC::LVX, dl, MVT::v16i8, MVT::Other, |
| 6274 |
{ZeroReg, N->getOperand(1), N->getOperand(0)}); |
6274 |
{ZeroReg, N->getOperand(1), N->getOperand(0)}); |
| 6275 |
|
6275 |
|
| 6276 |
SDNode *LoadHigh = LoadLow; |
6276 |
SDNode *LoadHigh = LoadLow; |
| 6277 |
if (Type == MVT::v8i16) { |
6277 |
if (Type == MVT::v8i16) { |
| 6278 |
LoadHigh = CurDAG->getMachineNode( |
6278 |
LoadHigh = CurDAG->getMachineNode( |
| 6279 |
PPC::LVX, dl, MVT::v16i8, MVT::Other, |
6279 |
PPC::LVX, dl, MVT::v16i8, MVT::Other, |
| 6280 |
{SDValue(CurDAG->getMachineNode( |
6280 |
{SDValue(CurDAG->getMachineNode( |
| 6281 |
LIOpcode, dl, MVT::i32, |
6281 |
LIOpcode, dl, MVT::i32, |
| 6282 |
CurDAG->getTargetConstant(1, dl, MVT::i8)), |
6282 |
CurDAG->getTargetConstant(1, dl, MVT::i8)), |
| 6283 |
0), |
6283 |
0), |
| 6284 |
N->getOperand(1), SDValue(LoadLow, 1)}); |
6284 |
N->getOperand(1), SDValue(LoadLow, 1)}); |
| 6285 |
} |
6285 |
} |
| 6286 |
|
6286 |
|
| 6287 |
CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), SDValue(LoadHigh, 1)); |
6287 |
CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), SDValue(LoadHigh, 1)); |
| 6288 |
transferMemOperands(N, LoadHigh); |
6288 |
transferMemOperands(N, LoadHigh); |
| 6289 |
|
6289 |
|
| 6290 |
SDNode *Perm = |
6290 |
SDNode *Perm = |
| 6291 |
CurDAG->getMachineNode(PPC::VPERM, dl, Type, SDValue(LoadLow, 0), |
6291 |
CurDAG->getMachineNode(PPC::VPERM, dl, Type, SDValue(LoadLow, 0), |
| 6292 |
SDValue(LoadHigh, 0), SDValue(Mask, 0)); |
6292 |
SDValue(LoadHigh, 0), SDValue(Mask, 0)); |
| 6293 |
CurDAG->SelectNodeTo(N, SplatOp, Type, |
6293 |
CurDAG->SelectNodeTo(N, SplatOp, Type, |
| 6294 |
CurDAG->getTargetConstant(SplatElemIndex, dl, MVT::i8), |
6294 |
CurDAG->getTargetConstant(SplatElemIndex, dl, MVT::i8), |
| 6295 |
SDValue(Perm, 0)); |
6295 |
SDValue(Perm, 0)); |
| 6296 |
return; |
6296 |
return; |
| 6297 |
} |
6297 |
} |
| 6298 |
} |
6298 |
} |
| 6299 |
|
6299 |
|
| 6300 |
SelectCode(N); |
6300 |
SelectCode(N); |
| 6301 |
} |
6301 |
} |
| 6302 |
|
6302 |
|
| 6303 |
// If the target supports the cmpb instruction, do the idiom recognition here. |
6303 |
// If the target supports the cmpb instruction, do the idiom recognition here. |
| 6304 |
// We don't do this as a DAG combine because we don't want to do it as nodes |
6304 |
// We don't do this as a DAG combine because we don't want to do it as nodes |
| 6305 |
// are being combined (because we might miss part of the eventual idiom). We |
6305 |
// are being combined (because we might miss part of the eventual idiom). We |
| 6306 |
// don't want to do it during instruction selection because we want to reuse |
6306 |
// don't want to do it during instruction selection because we want to reuse |
| 6307 |
// the logic for lowering the masking operations already part of the |
6307 |
// the logic for lowering the masking operations already part of the |
| 6308 |
// instruction selector. |
6308 |
// instruction selector. |
| 6309 |
SDValue PPCDAGToDAGISel::combineToCMPB(SDNode *N) { |
6309 |
SDValue PPCDAGToDAGISel::combineToCMPB(SDNode *N) { |
| 6310 |
SDLoc dl(N); |
6310 |
SDLoc dl(N); |
| 6311 |
|
6311 |
|
| 6312 |
assert(N->getOpcode() == ISD::OR && |
6312 |
assert(N->getOpcode() == ISD::OR && |
| 6313 |
"Only OR nodes are supported for CMPB"); |
6313 |
"Only OR nodes are supported for CMPB"); |
| 6314 |
|
6314 |
|
| 6315 |
SDValue Res; |
6315 |
SDValue Res; |
| 6316 |
if (!Subtarget->hasCMPB()) |
6316 |
if (!Subtarget->hasCMPB()) |
| 6317 |
return Res; |
6317 |
return Res; |
| 6318 |
|
6318 |
|
| 6319 |
if (N->getValueType(0) != MVT::i32 && |
6319 |
if (N->getValueType(0) != MVT::i32 && |
| 6320 |
N->getValueType(0) != MVT::i64) |
6320 |
N->getValueType(0) != MVT::i64) |
| 6321 |
return Res; |
6321 |
return Res; |
| 6322 |
|
6322 |
|
| 6323 |
EVT VT = N->getValueType(0); |
6323 |
EVT VT = N->getValueType(0); |
| 6324 |
|
6324 |
|
| 6325 |
SDValue RHS, LHS; |
6325 |
SDValue RHS, LHS; |
| 6326 |
bool BytesFound[8] = {false, false, false, false, false, false, false, false}; |
6326 |
bool BytesFound[8] = {false, false, false, false, false, false, false, false}; |
| 6327 |
uint64_t Mask = 0, Alt = 0; |
6327 |
uint64_t Mask = 0, Alt = 0; |
| 6328 |
|
6328 |
|
| 6329 |
auto IsByteSelectCC = [this](SDValue O, unsigned &b, |
6329 |
auto IsByteSelectCC = [this](SDValue O, unsigned &b, |
| 6330 |
uint64_t &Mask, uint64_t &Alt, |
6330 |
uint64_t &Mask, uint64_t &Alt, |
| 6331 |
SDValue &LHS, SDValue &RHS) { |
6331 |
SDValue &LHS, SDValue &RHS) { |
| 6332 |
if (O.getOpcode() != ISD::SELECT_CC) |
6332 |
if (O.getOpcode() != ISD::SELECT_CC) |
| 6333 |
return false; |
6333 |
return false; |
| 6334 |
ISD::CondCode CC = cast(O.getOperand(4))->get(); |
6334 |
ISD::CondCode CC = cast(O.getOperand(4))->get(); |
| 6335 |
|
6335 |
|
| 6336 |
if (!isa(O.getOperand(2)) || |
6336 |
if (!isa(O.getOperand(2)) || |
| 6337 |
!isa(O.getOperand(3))) |
6337 |
!isa(O.getOperand(3))) |
| 6338 |
return false; |
6338 |
return false; |
| 6339 |
|
6339 |
|
| 6340 |
uint64_t PM = O.getConstantOperandVal(2); |
6340 |
uint64_t PM = O.getConstantOperandVal(2); |
| 6341 |
uint64_t PAlt = O.getConstantOperandVal(3); |
6341 |
uint64_t PAlt = O.getConstantOperandVal(3); |
| 6342 |
for (b = 0; b < 8; ++b) { |
6342 |
for (b = 0; b < 8; ++b) { |
| 6343 |
uint64_t Mask = UINT64_C(0xFF) << (8*b); |
6343 |
uint64_t Mask = UINT64_C(0xFF) << (8*b); |
| 6344 |
if (PM && (PM & Mask) == PM && (PAlt & Mask) == PAlt) |
6344 |
if (PM && (PM & Mask) == PM && (PAlt & Mask) == PAlt) |
| 6345 |
break; |
6345 |
break; |
| 6346 |
} |
6346 |
} |
| 6347 |
|
6347 |
|
| 6348 |
if (b == 8) |
6348 |
if (b == 8) |
| 6349 |
return false; |
6349 |
return false; |
| 6350 |
Mask |= PM; |
6350 |
Mask |= PM; |
| 6351 |
Alt |= PAlt; |
6351 |
Alt |= PAlt; |
| 6352 |
|
6352 |
|
| 6353 |
if (!isa(O.getOperand(1)) || |
6353 |
if (!isa(O.getOperand(1)) || |
| 6354 |
O.getConstantOperandVal(1) != 0) { |
6354 |
O.getConstantOperandVal(1) != 0) { |
| 6355 |
SDValue Op0 = O.getOperand(0), Op1 = O.getOperand(1); |
6355 |
SDValue Op0 = O.getOperand(0), Op1 = O.getOperand(1); |
| 6356 |
if (Op0.getOpcode() == ISD::TRUNCATE) |
6356 |
if (Op0.getOpcode() == ISD::TRUNCATE) |
| 6357 |
Op0 = Op0.getOperand(0); |
6357 |
Op0 = Op0.getOperand(0); |
| 6358 |
if (Op1.getOpcode() == ISD::TRUNCATE) |
6358 |
if (Op1.getOpcode() == ISD::TRUNCATE) |
| 6359 |
Op1 = Op1.getOperand(0); |
6359 |
Op1 = Op1.getOperand(0); |
| 6360 |
|
6360 |
|
| 6361 |
if (Op0.getOpcode() == ISD::SRL && Op1.getOpcode() == ISD::SRL && |
6361 |
if (Op0.getOpcode() == ISD::SRL && Op1.getOpcode() == ISD::SRL && |
| 6362 |
Op0.getOperand(1) == Op1.getOperand(1) && CC == ISD::SETEQ && |
6362 |
Op0.getOperand(1) == Op1.getOperand(1) && CC == ISD::SETEQ && |
| 6363 |
isa(Op0.getOperand(1))) { |
6363 |
isa(Op0.getOperand(1))) { |
| 6364 |
|
6364 |
|
| 6365 |
unsigned Bits = Op0.getValueSizeInBits(); |
6365 |
unsigned Bits = Op0.getValueSizeInBits(); |
| 6366 |
if (b != Bits/8-1) |
6366 |
if (b != Bits/8-1) |
| 6367 |
return false; |
6367 |
return false; |
| 6368 |
if (Op0.getConstantOperandVal(1) != Bits-8) |
6368 |
if (Op0.getConstantOperandVal(1) != Bits-8) |
| 6369 |
return false; |
6369 |
return false; |
| 6370 |
|
6370 |
|
| 6371 |
LHS = Op0.getOperand(0); |
6371 |
LHS = Op0.getOperand(0); |
| 6372 |
RHS = Op1.getOperand(0); |
6372 |
RHS = Op1.getOperand(0); |
| 6373 |
return true; |
6373 |
return true; |
| 6374 |
} |
6374 |
} |
| 6375 |
|
6375 |
|
| 6376 |
// When we have small integers (i16 to be specific), the form present |
6376 |
// When we have small integers (i16 to be specific), the form present |
| 6377 |
// post-legalization uses SETULT in the SELECT_CC for the |
6377 |
// post-legalization uses SETULT in the SELECT_CC for the |
| 6378 |
// higher-order byte, depending on the fact that the |
6378 |
// higher-order byte, depending on the fact that the |
| 6379 |
// even-higher-order bytes are known to all be zero, for example: |
6379 |
// even-higher-order bytes are known to all be zero, for example: |
| 6380 |
// select_cc (xor $lhs, $rhs), 256, 65280, 0, setult |
6380 |
// select_cc (xor $lhs, $rhs), 256, 65280, 0, setult |
| 6381 |
// (so when the second byte is the same, because all higher-order |
6381 |
// (so when the second byte is the same, because all higher-order |
| 6382 |
// bits from bytes 3 and 4 are known to be zero, the result of the |
6382 |
// bits from bytes 3 and 4 are known to be zero, the result of the |
| 6383 |
// xor can be at most 255) |
6383 |
// xor can be at most 255) |
| 6384 |
if (Op0.getOpcode() == ISD::XOR && CC == ISD::SETULT && |
6384 |
if (Op0.getOpcode() == ISD::XOR && CC == ISD::SETULT && |
| 6385 |
isa(O.getOperand(1))) { |
6385 |
isa(O.getOperand(1))) { |
| 6386 |
|
6386 |
|
| 6387 |
uint64_t ULim = O.getConstantOperandVal(1); |
6387 |
uint64_t ULim = O.getConstantOperandVal(1); |
| 6388 |
if (ULim != (UINT64_C(1) << b*8)) |
6388 |
if (ULim != (UINT64_C(1) << b*8)) |
| 6389 |
return false; |
6389 |
return false; |
| 6390 |
|
6390 |
|
| 6391 |
// Now we need to make sure that the upper bytes are known to be |
6391 |
// Now we need to make sure that the upper bytes are known to be |
| 6392 |
// zero. |
6392 |
// zero. |
| 6393 |
unsigned Bits = Op0.getValueSizeInBits(); |
6393 |
unsigned Bits = Op0.getValueSizeInBits(); |
| 6394 |
if (!CurDAG->MaskedValueIsZero( |
6394 |
if (!CurDAG->MaskedValueIsZero( |
| 6395 |
Op0, APInt::getHighBitsSet(Bits, Bits - (b + 1) * 8))) |
6395 |
Op0, APInt::getHighBitsSet(Bits, Bits - (b + 1) * 8))) |
| 6396 |
return false; |
6396 |
return false; |
| 6397 |
|
6397 |
|
| 6398 |
LHS = Op0.getOperand(0); |
6398 |
LHS = Op0.getOperand(0); |
| 6399 |
RHS = Op0.getOperand(1); |
6399 |
RHS = Op0.getOperand(1); |
| 6400 |
return true; |
6400 |
return true; |
| 6401 |
} |
6401 |
} |
| 6402 |
|
6402 |
|
| 6403 |
return false; |
6403 |
return false; |
| 6404 |
} |
6404 |
} |
| 6405 |
|
6405 |
|
| 6406 |
if (CC != ISD::SETEQ) |
6406 |
if (CC != ISD::SETEQ) |
| 6407 |
return false; |
6407 |
return false; |
| 6408 |
|
6408 |
|
| 6409 |
SDValue Op = O.getOperand(0); |
6409 |
SDValue Op = O.getOperand(0); |
| 6410 |
if (Op.getOpcode() == ISD::AND) { |
6410 |
if (Op.getOpcode() == ISD::AND) { |
| 6411 |
if (!isa(Op.getOperand(1))) |
6411 |
if (!isa(Op.getOperand(1))) |
| 6412 |
return false; |
6412 |
return false; |
| 6413 |
if (Op.getConstantOperandVal(1) != (UINT64_C(0xFF) << (8*b))) |
6413 |
if (Op.getConstantOperandVal(1) != (UINT64_C(0xFF) << (8*b))) |
| 6414 |
return false; |
6414 |
return false; |
| 6415 |
|
6415 |
|
| 6416 |
SDValue XOR = Op.getOperand(0); |
6416 |
SDValue XOR = Op.getOperand(0); |
| 6417 |
if (XOR.getOpcode() == ISD::TRUNCATE) |
6417 |
if (XOR.getOpcode() == ISD::TRUNCATE) |
| 6418 |
XOR = XOR.getOperand(0); |
6418 |
XOR = XOR.getOperand(0); |
| 6419 |
if (XOR.getOpcode() != ISD::XOR) |
6419 |
if (XOR.getOpcode() != ISD::XOR) |
| 6420 |
return false; |
6420 |
return false; |
| 6421 |
|
6421 |
|
| 6422 |
LHS = XOR.getOperand(0); |
6422 |
LHS = XOR.getOperand(0); |
| 6423 |
RHS = XOR.getOperand(1); |
6423 |
RHS = XOR.getOperand(1); |
| 6424 |
return true; |
6424 |
return true; |
| 6425 |
} else if (Op.getOpcode() == ISD::SRL) { |
6425 |
} else if (Op.getOpcode() == ISD::SRL) { |
| 6426 |
if (!isa(Op.getOperand(1))) |
6426 |
if (!isa(Op.getOperand(1))) |
| 6427 |
return false; |
6427 |
return false; |
| 6428 |
unsigned Bits = Op.getValueSizeInBits(); |
6428 |
unsigned Bits = Op.getValueSizeInBits(); |
| 6429 |
if (b != Bits/8-1) |
6429 |
if (b != Bits/8-1) |
| 6430 |
return false; |
6430 |
return false; |
| 6431 |
if (Op.getConstantOperandVal(1) != Bits-8) |
6431 |
if (Op.getConstantOperandVal(1) != Bits-8) |
| 6432 |
return false; |
6432 |
return false; |
| 6433 |
|
6433 |
|
| 6434 |
SDValue XOR = Op.getOperand(0); |
6434 |
SDValue XOR = Op.getOperand(0); |
| 6435 |
if (XOR.getOpcode() == ISD::TRUNCATE) |
6435 |
if (XOR.getOpcode() == ISD::TRUNCATE) |
| 6436 |
XOR = XOR.getOperand(0); |
6436 |
XOR = XOR.getOperand(0); |
| 6437 |
if (XOR.getOpcode() != ISD::XOR) |
6437 |
if (XOR.getOpcode() != ISD::XOR) |
| 6438 |
return false; |
6438 |
return false; |
| 6439 |
|
6439 |
|
| 6440 |
LHS = XOR.getOperand(0); |
6440 |
LHS = XOR.getOperand(0); |
| 6441 |
RHS = XOR.getOperand(1); |
6441 |
RHS = XOR.getOperand(1); |
| 6442 |
return true; |
6442 |
return true; |
| 6443 |
} |
6443 |
} |
| 6444 |
|
6444 |
|
| 6445 |
return false; |
6445 |
return false; |
| 6446 |
}; |
6446 |
}; |
| 6447 |
|
6447 |
|
| 6448 |
SmallVector Queue(1, SDValue(N, 0)); |
6448 |
SmallVector Queue(1, SDValue(N, 0)); |
| 6449 |
while (!Queue.empty()) { |
6449 |
while (!Queue.empty()) { |
| 6450 |
SDValue V = Queue.pop_back_val(); |
6450 |
SDValue V = Queue.pop_back_val(); |
| 6451 |
|
6451 |
|
| 6452 |
for (const SDValue &O : V.getNode()->ops()) { |
6452 |
for (const SDValue &O : V.getNode()->ops()) { |
| 6453 |
unsigned b = 0; |
6453 |
unsigned b = 0; |
| 6454 |
uint64_t M = 0, A = 0; |
6454 |
uint64_t M = 0, A = 0; |
| 6455 |
SDValue OLHS, ORHS; |
6455 |
SDValue OLHS, ORHS; |
| 6456 |
if (O.getOpcode() == ISD::OR) { |
6456 |
if (O.getOpcode() == ISD::OR) { |
| 6457 |
Queue.push_back(O); |
6457 |
Queue.push_back(O); |
| 6458 |
} else if (IsByteSelectCC(O, b, M, A, OLHS, ORHS)) { |
6458 |
} else if (IsByteSelectCC(O, b, M, A, OLHS, ORHS)) { |
| 6459 |
if (!LHS) { |
6459 |
if (!LHS) { |
| 6460 |
LHS = OLHS; |
6460 |
LHS = OLHS; |
| 6461 |
RHS = ORHS; |
6461 |
RHS = ORHS; |
| 6462 |
BytesFound[b] = true; |
6462 |
BytesFound[b] = true; |
| 6463 |
Mask |= M; |
6463 |
Mask |= M; |
| 6464 |
Alt |= A; |
6464 |
Alt |= A; |
| 6465 |
} else if ((LHS == ORHS && RHS == OLHS) || |
6465 |
} else if ((LHS == ORHS && RHS == OLHS) || |
| 6466 |
(RHS == ORHS && LHS == OLHS)) { |
6466 |
(RHS == ORHS && LHS == OLHS)) { |
| 6467 |
BytesFound[b] = true; |
6467 |
BytesFound[b] = true; |
| 6468 |
Mask |= M; |
6468 |
Mask |= M; |
| 6469 |
Alt |= A; |
6469 |
Alt |= A; |
| 6470 |
} else { |
6470 |
} else { |
| 6471 |
return Res; |
6471 |
return Res; |
| 6472 |
} |
6472 |
} |
| 6473 |
} else { |
6473 |
} else { |
| 6474 |
return Res; |
6474 |
return Res; |
| 6475 |
} |
6475 |
} |
| 6476 |
} |
6476 |
} |
| 6477 |
} |
6477 |
} |
| 6478 |
|
6478 |
|
| 6479 |
unsigned LastB = 0, BCnt = 0; |
6479 |
unsigned LastB = 0, BCnt = 0; |
| 6480 |
for (unsigned i = 0; i < 8; ++i) |
6480 |
for (unsigned i = 0; i < 8; ++i) |
| 6481 |
if (BytesFound[LastB]) { |
6481 |
if (BytesFound[LastB]) { |
| 6482 |
++BCnt; |
6482 |
++BCnt; |
| 6483 |
LastB = i; |
6483 |
LastB = i; |
| 6484 |
} |
6484 |
} |
| 6485 |
|
6485 |
|
| 6486 |
if (!LastB || BCnt < 2) |
6486 |
if (!LastB || BCnt < 2) |
| 6487 |
return Res; |
6487 |
return Res; |
| 6488 |
|
6488 |
|
| 6489 |
// Because we'll be zero-extending the output anyway if don't have a specific |
6489 |
// Because we'll be zero-extending the output anyway if don't have a specific |
| 6490 |
// value for each input byte (via the Mask), we can 'anyext' the inputs. |
6490 |
// value for each input byte (via the Mask), we can 'anyext' the inputs. |
| 6491 |
if (LHS.getValueType() != VT) { |
6491 |
if (LHS.getValueType() != VT) { |
| 6492 |
LHS = CurDAG->getAnyExtOrTrunc(LHS, dl, VT); |
6492 |
LHS = CurDAG->getAnyExtOrTrunc(LHS, dl, VT); |
| 6493 |
RHS = CurDAG->getAnyExtOrTrunc(RHS, dl, VT); |
6493 |
RHS = CurDAG->getAnyExtOrTrunc(RHS, dl, VT); |
| 6494 |
} |
6494 |
} |
| 6495 |
|
6495 |
|
| 6496 |
Res = CurDAG->getNode(PPCISD::CMPB, dl, VT, LHS, RHS); |
6496 |
Res = CurDAG->getNode(PPCISD::CMPB, dl, VT, LHS, RHS); |
| 6497 |
|
6497 |
|
| 6498 |
bool NonTrivialMask = ((int64_t) Mask) != INT64_C(-1); |
6498 |
bool NonTrivialMask = ((int64_t) Mask) != INT64_C(-1); |
| 6499 |
if (NonTrivialMask && !Alt) { |
6499 |
if (NonTrivialMask && !Alt) { |
| 6500 |
// Res = Mask & CMPB |
6500 |
// Res = Mask & CMPB |
| 6501 |
Res = CurDAG->getNode(ISD::AND, dl, VT, Res, |
6501 |
Res = CurDAG->getNode(ISD::AND, dl, VT, Res, |
| 6502 |
CurDAG->getConstant(Mask, dl, VT)); |
6502 |
CurDAG->getConstant(Mask, dl, VT)); |
| 6503 |
} else if (Alt) { |
6503 |
} else if (Alt) { |
| 6504 |
// Res = (CMPB & Mask) | (~CMPB & Alt) |
6504 |
// Res = (CMPB & Mask) | (~CMPB & Alt) |
| 6505 |
// Which, as suggested here: |
6505 |
// Which, as suggested here: |
| 6506 |
// https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge |
6506 |
// https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge |
| 6507 |
// can be written as: |
6507 |
// can be written as: |
| 6508 |
// Res = Alt ^ ((Alt ^ Mask) & CMPB) |
6508 |
// Res = Alt ^ ((Alt ^ Mask) & CMPB) |
| 6509 |
// useful because the (Alt ^ Mask) can be pre-computed. |
6509 |
// useful because the (Alt ^ Mask) can be pre-computed. |
| 6510 |
Res = CurDAG->getNode(ISD::AND, dl, VT, Res, |
6510 |
Res = CurDAG->getNode(ISD::AND, dl, VT, Res, |
| 6511 |
CurDAG->getConstant(Mask ^ Alt, dl, VT)); |
6511 |
CurDAG->getConstant(Mask ^ Alt, dl, VT)); |
| 6512 |
Res = CurDAG->getNode(ISD::XOR, dl, VT, Res, |
6512 |
Res = CurDAG->getNode(ISD::XOR, dl, VT, Res, |
| 6513 |
CurDAG->getConstant(Alt, dl, VT)); |
6513 |
CurDAG->getConstant(Alt, dl, VT)); |
| 6514 |
} |
6514 |
} |
| 6515 |
|
6515 |
|
| 6516 |
return Res; |
6516 |
return Res; |
| 6517 |
} |
6517 |
} |
| 6518 |
|
6518 |
|
| 6519 |
// When CR bit registers are enabled, an extension of an i1 variable to a i32 |
6519 |
// When CR bit registers are enabled, an extension of an i1 variable to a i32 |
| 6520 |
// or i64 value is lowered in terms of a SELECT_I[48] operation, and thus |
6520 |
// or i64 value is lowered in terms of a SELECT_I[48] operation, and thus |
| 6521 |
// involves constant materialization of a 0 or a 1 or both. If the result of |
6521 |
// involves constant materialization of a 0 or a 1 or both. If the result of |
| 6522 |
// the extension is then operated upon by some operator that can be constant |
6522 |
// the extension is then operated upon by some operator that can be constant |
| 6523 |
// folded with a constant 0 or 1, and that constant can be materialized using |
6523 |
// folded with a constant 0 or 1, and that constant can be materialized using |
| 6524 |
// only one instruction (like a zero or one), then we should fold in those |
6524 |
// only one instruction (like a zero or one), then we should fold in those |
| 6525 |
// operations with the select. |
6525 |
// operations with the select. |
| 6526 |
void PPCDAGToDAGISel::foldBoolExts(SDValue &Res, SDNode *&N) { |
6526 |
void PPCDAGToDAGISel::foldBoolExts(SDValue &Res, SDNode *&N) { |
| 6527 |
if (!Subtarget->useCRBits()) |
6527 |
if (!Subtarget->useCRBits()) |
| 6528 |
return; |
6528 |
return; |
| 6529 |
|
6529 |
|
| 6530 |
if (N->getOpcode() != ISD::ZERO_EXTEND && |
6530 |
if (N->getOpcode() != ISD::ZERO_EXTEND && |
| 6531 |
N->getOpcode() != ISD::SIGN_EXTEND && |
6531 |
N->getOpcode() != ISD::SIGN_EXTEND && |
| 6532 |
N->getOpcode() != ISD::ANY_EXTEND) |
6532 |
N->getOpcode() != ISD::ANY_EXTEND) |
| 6533 |
return; |
6533 |
return; |
| 6534 |
|
6534 |
|
| 6535 |
if (N->getOperand(0).getValueType() != MVT::i1) |
6535 |
if (N->getOperand(0).getValueType() != MVT::i1) |
| 6536 |
return; |
6536 |
return; |
| 6537 |
|
6537 |
|
| 6538 |
if (!N->hasOneUse()) |
6538 |
if (!N->hasOneUse()) |
| 6539 |
return; |
6539 |
return; |
| 6540 |
|
6540 |
|
| 6541 |
SDLoc dl(N); |
6541 |
SDLoc dl(N); |
| 6542 |
EVT VT = N->getValueType(0); |
6542 |
EVT VT = N->getValueType(0); |
| 6543 |
SDValue Cond = N->getOperand(0); |
6543 |
SDValue Cond = N->getOperand(0); |
| 6544 |
SDValue ConstTrue = |
6544 |
SDValue ConstTrue = |
| 6545 |
CurDAG->getConstant(N->getOpcode() == ISD::SIGN_EXTEND ? -1 : 1, dl, VT); |
6545 |
CurDAG->getConstant(N->getOpcode() == ISD::SIGN_EXTEND ? -1 : 1, dl, VT); |
| 6546 |
SDValue ConstFalse = CurDAG->getConstant(0, dl, VT); |
6546 |
SDValue ConstFalse = CurDAG->getConstant(0, dl, VT); |
| 6547 |
|
6547 |
|
| 6548 |
do { |
6548 |
do { |
| 6549 |
SDNode *User = *N->use_begin(); |
6549 |
SDNode *User = *N->use_begin(); |
| 6550 |
if (User->getNumOperands() != 2) |
6550 |
if (User->getNumOperands() != 2) |
| 6551 |
break; |
6551 |
break; |
| 6552 |
|
6552 |
|
| 6553 |
auto TryFold = [this, N, User, dl](SDValue Val) { |
6553 |
auto TryFold = [this, N, User, dl](SDValue Val) { |
| 6554 |
SDValue UserO0 = User->getOperand(0), UserO1 = User->getOperand(1); |
6554 |
SDValue UserO0 = User->getOperand(0), UserO1 = User->getOperand(1); |
| 6555 |
SDValue O0 = UserO0.getNode() == N ? Val : UserO0; |
6555 |
SDValue O0 = UserO0.getNode() == N ? Val : UserO0; |
| 6556 |
SDValue O1 = UserO1.getNode() == N ? Val : UserO1; |
6556 |
SDValue O1 = UserO1.getNode() == N ? Val : UserO1; |
| 6557 |
|
6557 |
|
| 6558 |
return CurDAG->FoldConstantArithmetic(User->getOpcode(), dl, |
6558 |
return CurDAG->FoldConstantArithmetic(User->getOpcode(), dl, |
| 6559 |
User->getValueType(0), {O0, O1}); |
6559 |
User->getValueType(0), {O0, O1}); |
| 6560 |
}; |
6560 |
}; |
| 6561 |
|
6561 |
|
| 6562 |
// FIXME: When the semantics of the interaction between select and undef |
6562 |
// FIXME: When the semantics of the interaction between select and undef |
| 6563 |
// are clearly defined, it may turn out to be unnecessary to break here. |
6563 |
// are clearly defined, it may turn out to be unnecessary to break here. |
| 6564 |
SDValue TrueRes = TryFold(ConstTrue); |
6564 |
SDValue TrueRes = TryFold(ConstTrue); |
| 6565 |
if (!TrueRes || TrueRes.isUndef()) |
6565 |
if (!TrueRes || TrueRes.isUndef()) |
| 6566 |
break; |
6566 |
break; |
| 6567 |
SDValue FalseRes = TryFold(ConstFalse); |
6567 |
SDValue FalseRes = TryFold(ConstFalse); |
| 6568 |
if (!FalseRes || FalseRes.isUndef()) |
6568 |
if (!FalseRes || FalseRes.isUndef()) |
| 6569 |
break; |
6569 |
break; |
| 6570 |
|
6570 |
|
| 6571 |
// For us to materialize these using one instruction, we must be able to |
6571 |
// For us to materialize these using one instruction, we must be able to |
| 6572 |
// represent them as signed 16-bit integers. |
6572 |
// represent them as signed 16-bit integers. |
| 6573 |
uint64_t True = cast(TrueRes)->getZExtValue(), |
6573 |
uint64_t True = cast(TrueRes)->getZExtValue(), |
| 6574 |
False = cast(FalseRes)->getZExtValue(); |
6574 |
False = cast(FalseRes)->getZExtValue(); |
| 6575 |
if (!isInt<16>(True) || !isInt<16>(False)) |
6575 |
if (!isInt<16>(True) || !isInt<16>(False)) |
| 6576 |
break; |
6576 |
break; |
| 6577 |
|
6577 |
|
| 6578 |
// We can replace User with a new SELECT node, and try again to see if we |
6578 |
// We can replace User with a new SELECT node, and try again to see if we |
| 6579 |
// can fold the select with its user. |
6579 |
// can fold the select with its user. |
| 6580 |
Res = CurDAG->getSelect(dl, User->getValueType(0), Cond, TrueRes, FalseRes); |
6580 |
Res = CurDAG->getSelect(dl, User->getValueType(0), Cond, TrueRes, FalseRes); |
| 6581 |
N = User; |
6581 |
N = User; |
| 6582 |
ConstTrue = TrueRes; |
6582 |
ConstTrue = TrueRes; |
| 6583 |
ConstFalse = FalseRes; |
6583 |
ConstFalse = FalseRes; |
| 6584 |
} while (N->hasOneUse()); |
6584 |
} while (N->hasOneUse()); |
| 6585 |
} |
6585 |
} |
| 6586 |
|
6586 |
|
| 6587 |
void PPCDAGToDAGISel::PreprocessISelDAG() { |
6587 |
void PPCDAGToDAGISel::PreprocessISelDAG() { |
| 6588 |
SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); |
6588 |
SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); |
| 6589 |
|
6589 |
|
| 6590 |
bool MadeChange = false; |
6590 |
bool MadeChange = false; |
| 6591 |
while (Position != CurDAG->allnodes_begin()) { |
6591 |
while (Position != CurDAG->allnodes_begin()) { |
| 6592 |
SDNode *N = &*--Position; |
6592 |
SDNode *N = &*--Position; |
| 6593 |
if (N->use_empty()) |
6593 |
if (N->use_empty()) |
| 6594 |
continue; |
6594 |
continue; |
| 6595 |
|
6595 |
|
| 6596 |
SDValue Res; |
6596 |
SDValue Res; |
| 6597 |
switch (N->getOpcode()) { |
6597 |
switch (N->getOpcode()) { |
| 6598 |
default: break; |
6598 |
default: break; |
| 6599 |
case ISD::OR: |
6599 |
case ISD::OR: |
| 6600 |
Res = combineToCMPB(N); |
6600 |
Res = combineToCMPB(N); |
| 6601 |
break; |
6601 |
break; |
| 6602 |
} |
6602 |
} |
| 6603 |
|
6603 |
|
| 6604 |
if (!Res) |
6604 |
if (!Res) |
| 6605 |
foldBoolExts(Res, N); |
6605 |
foldBoolExts(Res, N); |
| 6606 |
|
6606 |
|
| 6607 |
if (Res) { |
6607 |
if (Res) { |
| 6608 |
LLVM_DEBUG(dbgs() << "PPC DAG preprocessing replacing:\nOld: "); |
6608 |
LLVM_DEBUG(dbgs() << "PPC DAG preprocessing replacing:\nOld: "); |
| 6609 |
LLVM_DEBUG(N->dump(CurDAG)); |
6609 |
LLVM_DEBUG(N->dump(CurDAG)); |
| 6610 |
LLVM_DEBUG(dbgs() << "\nNew: "); |
6610 |
LLVM_DEBUG(dbgs() << "\nNew: "); |
| 6611 |
LLVM_DEBUG(Res.getNode()->dump(CurDAG)); |
6611 |
LLVM_DEBUG(Res.getNode()->dump(CurDAG)); |
| 6612 |
LLVM_DEBUG(dbgs() << "\n"); |
6612 |
LLVM_DEBUG(dbgs() << "\n"); |
| 6613 |
|
6613 |
|
| 6614 |
CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); |
6614 |
CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); |
| 6615 |
MadeChange = true; |
6615 |
MadeChange = true; |
| 6616 |
} |
6616 |
} |
| 6617 |
} |
6617 |
} |
| 6618 |
|
6618 |
|
| 6619 |
if (MadeChange) |
6619 |
if (MadeChange) |
| 6620 |
CurDAG->RemoveDeadNodes(); |
6620 |
CurDAG->RemoveDeadNodes(); |
| 6621 |
} |
6621 |
} |
| 6622 |
|
6622 |
|
| 6623 |
/// PostprocessISelDAG - Perform some late peephole optimizations |
6623 |
/// PostprocessISelDAG - Perform some late peephole optimizations |
| 6624 |
/// on the DAG representation. |
6624 |
/// on the DAG representation. |
| 6625 |
void PPCDAGToDAGISel::PostprocessISelDAG() { |
6625 |
void PPCDAGToDAGISel::PostprocessISelDAG() { |
| 6626 |
// Skip peepholes at -O0. |
6626 |
// Skip peepholes at -O0. |
| 6627 |
if (TM.getOptLevel() == CodeGenOpt::None) |
6627 |
if (TM.getOptLevel() == CodeGenOpt::None) |
| 6628 |
return; |
6628 |
return; |
| 6629 |
|
6629 |
|
| 6630 |
PeepholePPC64(); |
6630 |
PeepholePPC64(); |
| 6631 |
PeepholeCROps(); |
6631 |
PeepholeCROps(); |
| 6632 |
PeepholePPC64ZExt(); |
6632 |
PeepholePPC64ZExt(); |
| 6633 |
} |
6633 |
} |
| 6634 |
|
6634 |
|
| 6635 |
// Check if all users of this node will become isel where the second operand |
6635 |
// Check if all users of this node will become isel where the second operand |
| 6636 |
// is the constant zero. If this is so, and if we can negate the condition, |
6636 |
// is the constant zero. If this is so, and if we can negate the condition, |
| 6637 |
// then we can flip the true and false operands. This will allow the zero to |
6637 |
// then we can flip the true and false operands. This will allow the zero to |
| 6638 |
// be folded with the isel so that we don't need to materialize a register |
6638 |
// be folded with the isel so that we don't need to materialize a register |
| 6639 |
// containing zero. |
6639 |
// containing zero. |
| 6640 |
bool PPCDAGToDAGISel::AllUsersSelectZero(SDNode *N) { |
6640 |
bool PPCDAGToDAGISel::AllUsersSelectZero(SDNode *N) { |
| 6641 |
for (const SDNode *User : N->uses()) { |
6641 |
for (const SDNode *User : N->uses()) { |
| 6642 |
if (!User->isMachineOpcode()) |
6642 |
if (!User->isMachineOpcode()) |
| 6643 |
return false; |
6643 |
return false; |
| 6644 |
if (User->getMachineOpcode() != PPC::SELECT_I4 && |
6644 |
if (User->getMachineOpcode() != PPC::SELECT_I4 && |
| 6645 |
User->getMachineOpcode() != PPC::SELECT_I8) |
6645 |
User->getMachineOpcode() != PPC::SELECT_I8) |
| 6646 |
return false; |
6646 |
return false; |
| 6647 |
|
6647 |
|
| 6648 |
SDNode *Op1 = User->getOperand(1).getNode(); |
6648 |
SDNode *Op1 = User->getOperand(1).getNode(); |
| 6649 |
SDNode *Op2 = User->getOperand(2).getNode(); |
6649 |
SDNode *Op2 = User->getOperand(2).getNode(); |
| 6650 |
// If we have a degenerate select with two equal operands, swapping will |
6650 |
// If we have a degenerate select with two equal operands, swapping will |
| 6651 |
// not do anything, and we may run into an infinite loop. |
6651 |
// not do anything, and we may run into an infinite loop. |
| 6652 |
if (Op1 == Op2) |
6652 |
if (Op1 == Op2) |
| 6653 |
return false; |
6653 |
return false; |
| 6654 |
|
6654 |
|
| 6655 |
if (!Op2->isMachineOpcode()) |
6655 |
if (!Op2->isMachineOpcode()) |
| 6656 |
return false; |
6656 |
return false; |
| 6657 |
|
6657 |
|
| 6658 |
if (Op2->getMachineOpcode() != PPC::LI && |
6658 |
if (Op2->getMachineOpcode() != PPC::LI && |
| 6659 |
Op2->getMachineOpcode() != PPC::LI8) |
6659 |
Op2->getMachineOpcode() != PPC::LI8) |
| 6660 |
return false; |
6660 |
return false; |
| 6661 |
|
6661 |
|
| 6662 |
ConstantSDNode *C = dyn_cast(Op2->getOperand(0)); |
6662 |
ConstantSDNode *C = dyn_cast(Op2->getOperand(0)); |
| 6663 |
if (!C) |
6663 |
if (!C) |
| 6664 |
return false; |
6664 |
return false; |
| 6665 |
|
6665 |
|
| 6666 |
if (!C->isZero()) |
6666 |
if (!C->isZero()) |
| 6667 |
return false; |
6667 |
return false; |
| 6668 |
} |
6668 |
} |
| 6669 |
|
6669 |
|
| 6670 |
return true; |
6670 |
return true; |
| 6671 |
} |
6671 |
} |
| 6672 |
|
6672 |
|
| 6673 |
void PPCDAGToDAGISel::SwapAllSelectUsers(SDNode *N) { |
6673 |
void PPCDAGToDAGISel::SwapAllSelectUsers(SDNode *N) { |
| 6674 |
SmallVector ToReplace; |
6674 |
SmallVector ToReplace; |
| 6675 |
for (SDNode *User : N->uses()) { |
6675 |
for (SDNode *User : N->uses()) { |
| 6676 |
assert((User->getMachineOpcode() == PPC::SELECT_I4 || |
6676 |
assert((User->getMachineOpcode() == PPC::SELECT_I4 || |
| 6677 |
User->getMachineOpcode() == PPC::SELECT_I8) && |
6677 |
User->getMachineOpcode() == PPC::SELECT_I8) && |
| 6678 |
"Must have all select users"); |
6678 |
"Must have all select users"); |
| 6679 |
ToReplace.push_back(User); |
6679 |
ToReplace.push_back(User); |
| 6680 |
} |
6680 |
} |
| 6681 |
|
6681 |
|
| 6682 |
for (SDNode *User : ToReplace) { |
6682 |
for (SDNode *User : ToReplace) { |
| 6683 |
SDNode *ResNode = |
6683 |
SDNode *ResNode = |
| 6684 |
CurDAG->getMachineNode(User->getMachineOpcode(), SDLoc(User), |
6684 |
CurDAG->getMachineNode(User->getMachineOpcode(), SDLoc(User), |
| 6685 |
User->getValueType(0), User->getOperand(0), |
6685 |
User->getValueType(0), User->getOperand(0), |
| 6686 |
User->getOperand(2), |
6686 |
User->getOperand(2), |
| 6687 |
User->getOperand(1)); |
6687 |
User->getOperand(1)); |
| 6688 |
|
6688 |
|
| 6689 |
LLVM_DEBUG(dbgs() << "CR Peephole replacing:\nOld: "); |
6689 |
LLVM_DEBUG(dbgs() << "CR Peephole replacing:\nOld: "); |
| 6690 |
LLVM_DEBUG(User->dump(CurDAG)); |
6690 |
LLVM_DEBUG(User->dump(CurDAG)); |
| 6691 |
LLVM_DEBUG(dbgs() << "\nNew: "); |
6691 |
LLVM_DEBUG(dbgs() << "\nNew: "); |
| 6692 |
LLVM_DEBUG(ResNode->dump(CurDAG)); |
6692 |
LLVM_DEBUG(ResNode->dump(CurDAG)); |
| 6693 |
LLVM_DEBUG(dbgs() << "\n"); |
6693 |
LLVM_DEBUG(dbgs() << "\n"); |
| 6694 |
|
6694 |
|
| 6695 |
ReplaceUses(User, ResNode); |
6695 |
ReplaceUses(User, ResNode); |
| 6696 |
} |
6696 |
} |
| 6697 |
} |
6697 |
} |
| 6698 |
|
6698 |
|
| 6699 |
void PPCDAGToDAGISel::PeepholeCROps() { |
6699 |
void PPCDAGToDAGISel::PeepholeCROps() { |
| 6700 |
bool IsModified; |
6700 |
bool IsModified; |
| 6701 |
do { |
6701 |
do { |
| 6702 |
IsModified = false; |
6702 |
IsModified = false; |
| 6703 |
for (SDNode &Node : CurDAG->allnodes()) { |
6703 |
for (SDNode &Node : CurDAG->allnodes()) { |
| 6704 |
MachineSDNode *MachineNode = dyn_cast(&Node); |
6704 |
MachineSDNode *MachineNode = dyn_cast(&Node); |
| 6705 |
if (!MachineNode || MachineNode->use_empty()) |
6705 |
if (!MachineNode || MachineNode->use_empty()) |
| 6706 |
continue; |
6706 |
continue; |
| 6707 |
SDNode *ResNode = MachineNode; |
6707 |
SDNode *ResNode = MachineNode; |
| 6708 |
|
6708 |
|
| 6709 |
bool Op1Set = false, Op1Unset = false, |
6709 |
bool Op1Set = false, Op1Unset = false, |
| 6710 |
Op1Not = false, |
6710 |
Op1Not = false, |
| 6711 |
Op2Set = false, Op2Unset = false, |
6711 |
Op2Set = false, Op2Unset = false, |
| 6712 |
Op2Not = false; |
6712 |
Op2Not = false; |
| 6713 |
|
6713 |
|
| 6714 |
unsigned Opcode = MachineNode->getMachineOpcode(); |
6714 |
unsigned Opcode = MachineNode->getMachineOpcode(); |
| 6715 |
switch (Opcode) { |
6715 |
switch (Opcode) { |
| 6716 |
default: break; |
6716 |
default: break; |
| 6717 |
case PPC::CRAND: |
6717 |
case PPC::CRAND: |
| 6718 |
case PPC::CRNAND: |
6718 |
case PPC::CRNAND: |
| 6719 |
case PPC::CROR: |
6719 |
case PPC::CROR: |
| 6720 |
case PPC::CRXOR: |
6720 |
case PPC::CRXOR: |
| 6721 |
case PPC::CRNOR: |
6721 |
case PPC::CRNOR: |
| 6722 |
case PPC::CREQV: |
6722 |
case PPC::CREQV: |
| 6723 |
case PPC::CRANDC: |
6723 |
case PPC::CRANDC: |
| 6724 |
case PPC::CRORC: { |
6724 |
case PPC::CRORC: { |
| 6725 |
SDValue Op = MachineNode->getOperand(1); |
6725 |
SDValue Op = MachineNode->getOperand(1); |
| 6726 |
if (Op.isMachineOpcode()) { |
6726 |
if (Op.isMachineOpcode()) { |
| 6727 |
if (Op.getMachineOpcode() == PPC::CRSET) |
6727 |
if (Op.getMachineOpcode() == PPC::CRSET) |
| 6728 |
Op2Set = true; |
6728 |
Op2Set = true; |
| 6729 |
else if (Op.getMachineOpcode() == PPC::CRUNSET) |
6729 |
else if (Op.getMachineOpcode() == PPC::CRUNSET) |
| 6730 |
Op2Unset = true; |
6730 |
Op2Unset = true; |
| 6731 |
else if ((Op.getMachineOpcode() == PPC::CRNOR && |
6731 |
else if ((Op.getMachineOpcode() == PPC::CRNOR && |
| 6732 |
Op.getOperand(0) == Op.getOperand(1)) || |
6732 |
Op.getOperand(0) == Op.getOperand(1)) || |
| 6733 |
Op.getMachineOpcode() == PPC::CRNOT) |
6733 |
Op.getMachineOpcode() == PPC::CRNOT) |
| 6734 |
Op2Not = true; |
6734 |
Op2Not = true; |
| 6735 |
} |
6735 |
} |
| 6736 |
[[fallthrough]]; |
6736 |
[[fallthrough]]; |
| 6737 |
} |
6737 |
} |
| 6738 |
case PPC::BC: |
6738 |
case PPC::BC: |
| 6739 |
case PPC::BCn: |
6739 |
case PPC::BCn: |
| 6740 |
case PPC::SELECT_I4: |
6740 |
case PPC::SELECT_I4: |
| 6741 |
case PPC::SELECT_I8: |
6741 |
case PPC::SELECT_I8: |
| 6742 |
case PPC::SELECT_F4: |
6742 |
case PPC::SELECT_F4: |
| 6743 |
case PPC::SELECT_F8: |
6743 |
case PPC::SELECT_F8: |
| 6744 |
case PPC::SELECT_SPE: |
6744 |
case PPC::SELECT_SPE: |
| 6745 |
case PPC::SELECT_SPE4: |
6745 |
case PPC::SELECT_SPE4: |
| 6746 |
case PPC::SELECT_VRRC: |
6746 |
case PPC::SELECT_VRRC: |
| 6747 |
case PPC::SELECT_VSFRC: |
6747 |
case PPC::SELECT_VSFRC: |
| 6748 |
case PPC::SELECT_VSSRC: |
6748 |
case PPC::SELECT_VSSRC: |
| 6749 |
case PPC::SELECT_VSRC: { |
6749 |
case PPC::SELECT_VSRC: { |
| 6750 |
SDValue Op = MachineNode->getOperand(0); |
6750 |
SDValue Op = MachineNode->getOperand(0); |
| 6751 |
if (Op.isMachineOpcode()) { |
6751 |
if (Op.isMachineOpcode()) { |
| 6752 |
if (Op.getMachineOpcode() == PPC::CRSET) |
6752 |
if (Op.getMachineOpcode() == PPC::CRSET) |
| 6753 |
Op1Set = true; |
6753 |
Op1Set = true; |
| 6754 |
else if (Op.getMachineOpcode() == PPC::CRUNSET) |
6754 |
else if (Op.getMachineOpcode() == PPC::CRUNSET) |
| 6755 |
Op1Unset = true; |
6755 |
Op1Unset = true; |
| 6756 |
else if ((Op.getMachineOpcode() == PPC::CRNOR && |
6756 |
else if ((Op.getMachineOpcode() == PPC::CRNOR && |
| 6757 |
Op.getOperand(0) == Op.getOperand(1)) || |
6757 |
Op.getOperand(0) == Op.getOperand(1)) || |
| 6758 |
Op.getMachineOpcode() == PPC::CRNOT) |
6758 |
Op.getMachineOpcode() == PPC::CRNOT) |
| 6759 |
Op1Not = true; |
6759 |
Op1Not = true; |
| 6760 |
} |
6760 |
} |
| 6761 |
} |
6761 |
} |
| 6762 |
break; |
6762 |
break; |
| 6763 |
} |
6763 |
} |
| 6764 |
|
6764 |
|
| 6765 |
bool SelectSwap = false; |
6765 |
bool SelectSwap = false; |
| 6766 |
switch (Opcode) { |
6766 |
switch (Opcode) { |
| 6767 |
default: break; |
6767 |
default: break; |
| 6768 |
case PPC::CRAND: |
6768 |
case PPC::CRAND: |
| 6769 |
if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) |
6769 |
if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) |
| 6770 |
// x & x = x |
6770 |
// x & x = x |
| 6771 |
ResNode = MachineNode->getOperand(0).getNode(); |
6771 |
ResNode = MachineNode->getOperand(0).getNode(); |
| 6772 |
else if (Op1Set) |
6772 |
else if (Op1Set) |
| 6773 |
// 1 & y = y |
6773 |
// 1 & y = y |
| 6774 |
ResNode = MachineNode->getOperand(1).getNode(); |
6774 |
ResNode = MachineNode->getOperand(1).getNode(); |
| 6775 |
else if (Op2Set) |
6775 |
else if (Op2Set) |
| 6776 |
// x & 1 = x |
6776 |
// x & 1 = x |
| 6777 |
ResNode = MachineNode->getOperand(0).getNode(); |
6777 |
ResNode = MachineNode->getOperand(0).getNode(); |
| 6778 |
else if (Op1Unset || Op2Unset) |
6778 |
else if (Op1Unset || Op2Unset) |
| 6779 |
// x & 0 = 0 & y = 0 |
6779 |
// x & 0 = 0 & y = 0 |
| 6780 |
ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
6780 |
ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
| 6781 |
MVT::i1); |
6781 |
MVT::i1); |
| 6782 |
else if (Op1Not) |
6782 |
else if (Op1Not) |
| 6783 |
// ~x & y = andc(y, x) |
6783 |
// ~x & y = andc(y, x) |
| 6784 |
ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
6784 |
ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
| 6785 |
MVT::i1, MachineNode->getOperand(1), |
6785 |
MVT::i1, MachineNode->getOperand(1), |
| 6786 |
MachineNode->getOperand(0). |
6786 |
MachineNode->getOperand(0). |
| 6787 |
getOperand(0)); |
6787 |
getOperand(0)); |
| 6788 |
else if (Op2Not) |
6788 |
else if (Op2Not) |
| 6789 |
// x & ~y = andc(x, y) |
6789 |
// x & ~y = andc(x, y) |
| 6790 |
ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
6790 |
ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
| 6791 |
MVT::i1, MachineNode->getOperand(0), |
6791 |
MVT::i1, MachineNode->getOperand(0), |
| 6792 |
MachineNode->getOperand(1). |
6792 |
MachineNode->getOperand(1). |
| 6793 |
getOperand(0)); |
6793 |
getOperand(0)); |
| 6794 |
else if (AllUsersSelectZero(MachineNode)) { |
6794 |
else if (AllUsersSelectZero(MachineNode)) { |
| 6795 |
ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode), |
6795 |
ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode), |
| 6796 |
MVT::i1, MachineNode->getOperand(0), |
6796 |
MVT::i1, MachineNode->getOperand(0), |
| 6797 |
MachineNode->getOperand(1)); |
6797 |
MachineNode->getOperand(1)); |
| 6798 |
SelectSwap = true; |
6798 |
SelectSwap = true; |
| 6799 |
} |
6799 |
} |
| 6800 |
break; |
6800 |
break; |
| 6801 |
case PPC::CRNAND: |
6801 |
case PPC::CRNAND: |
| 6802 |
if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) |
6802 |
if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) |
| 6803 |
// nand(x, x) -> nor(x, x) |
6803 |
// nand(x, x) -> nor(x, x) |
| 6804 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6804 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
| 6805 |
MVT::i1, MachineNode->getOperand(0), |
6805 |
MVT::i1, MachineNode->getOperand(0), |
| 6806 |
MachineNode->getOperand(0)); |
6806 |
MachineNode->getOperand(0)); |
| 6807 |
else if (Op1Set) |
6807 |
else if (Op1Set) |
| 6808 |
// nand(1, y) -> nor(y, y) |
6808 |
// nand(1, y) -> nor(y, y) |
| 6809 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6809 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
| 6810 |
MVT::i1, MachineNode->getOperand(1), |
6810 |
MVT::i1, MachineNode->getOperand(1), |
| 6811 |
MachineNode->getOperand(1)); |
6811 |
MachineNode->getOperand(1)); |
| 6812 |
else if (Op2Set) |
6812 |
else if (Op2Set) |
| 6813 |
// nand(x, 1) -> nor(x, x) |
6813 |
// nand(x, 1) -> nor(x, x) |
| 6814 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6814 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
| 6815 |
MVT::i1, MachineNode->getOperand(0), |
6815 |
MVT::i1, MachineNode->getOperand(0), |
| 6816 |
MachineNode->getOperand(0)); |
6816 |
MachineNode->getOperand(0)); |
| 6817 |
else if (Op1Unset || Op2Unset) |
6817 |
else if (Op1Unset || Op2Unset) |
| 6818 |
// nand(x, 0) = nand(0, y) = 1 |
6818 |
// nand(x, 0) = nand(0, y) = 1 |
| 6819 |
ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
6819 |
ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
| 6820 |
MVT::i1); |
6820 |
MVT::i1); |
| 6821 |
else if (Op1Not) |
6821 |
else if (Op1Not) |
| 6822 |
// nand(~x, y) = ~(~x & y) = x | ~y = orc(x, y) |
6822 |
// nand(~x, y) = ~(~x & y) = x | ~y = orc(x, y) |
| 6823 |
ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
6823 |
ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
| 6824 |
MVT::i1, MachineNode->getOperand(0). |
6824 |
MVT::i1, MachineNode->getOperand(0). |
| 6825 |
getOperand(0), |
6825 |
getOperand(0), |
| 6826 |
MachineNode->getOperand(1)); |
6826 |
MachineNode->getOperand(1)); |
| 6827 |
else if (Op2Not) |
6827 |
else if (Op2Not) |
| 6828 |
// nand(x, ~y) = ~x | y = orc(y, x) |
6828 |
// nand(x, ~y) = ~x | y = orc(y, x) |
| 6829 |
ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
6829 |
ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
| 6830 |
MVT::i1, MachineNode->getOperand(1). |
6830 |
MVT::i1, MachineNode->getOperand(1). |
| 6831 |
getOperand(0), |
6831 |
getOperand(0), |
| 6832 |
MachineNode->getOperand(0)); |
6832 |
MachineNode->getOperand(0)); |
| 6833 |
else if (AllUsersSelectZero(MachineNode)) { |
6833 |
else if (AllUsersSelectZero(MachineNode)) { |
| 6834 |
ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode), |
6834 |
ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode), |
| 6835 |
MVT::i1, MachineNode->getOperand(0), |
6835 |
MVT::i1, MachineNode->getOperand(0), |
| 6836 |
MachineNode->getOperand(1)); |
6836 |
MachineNode->getOperand(1)); |
| 6837 |
SelectSwap = true; |
6837 |
SelectSwap = true; |
| 6838 |
} |
6838 |
} |
| 6839 |
break; |
6839 |
break; |
| 6840 |
case PPC::CROR: |
6840 |
case PPC::CROR: |
| 6841 |
if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) |
6841 |
if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) |
| 6842 |
// x | x = x |
6842 |
// x | x = x |
| 6843 |
ResNode = MachineNode->getOperand(0).getNode(); |
6843 |
ResNode = MachineNode->getOperand(0).getNode(); |
| 6844 |
else if (Op1Set || Op2Set) |
6844 |
else if (Op1Set || Op2Set) |
| 6845 |
// x | 1 = 1 | y = 1 |
6845 |
// x | 1 = 1 | y = 1 |
| 6846 |
ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
6846 |
ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
| 6847 |
MVT::i1); |
6847 |
MVT::i1); |
| 6848 |
else if (Op1Unset) |
6848 |
else if (Op1Unset) |
| 6849 |
// 0 | y = y |
6849 |
// 0 | y = y |
| 6850 |
ResNode = MachineNode->getOperand(1).getNode(); |
6850 |
ResNode = MachineNode->getOperand(1).getNode(); |
| 6851 |
else if (Op2Unset) |
6851 |
else if (Op2Unset) |
| 6852 |
// x | 0 = x |
6852 |
// x | 0 = x |
| 6853 |
ResNode = MachineNode->getOperand(0).getNode(); |
6853 |
ResNode = MachineNode->getOperand(0).getNode(); |
| 6854 |
else if (Op1Not) |
6854 |
else if (Op1Not) |
| 6855 |
// ~x | y = orc(y, x) |
6855 |
// ~x | y = orc(y, x) |
| 6856 |
ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
6856 |
ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
| 6857 |
MVT::i1, MachineNode->getOperand(1), |
6857 |
MVT::i1, MachineNode->getOperand(1), |
| 6858 |
MachineNode->getOperand(0). |
6858 |
MachineNode->getOperand(0). |
| 6859 |
getOperand(0)); |
6859 |
getOperand(0)); |
| 6860 |
else if (Op2Not) |
6860 |
else if (Op2Not) |
| 6861 |
// x | ~y = orc(x, y) |
6861 |
// x | ~y = orc(x, y) |
| 6862 |
ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
6862 |
ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
| 6863 |
MVT::i1, MachineNode->getOperand(0), |
6863 |
MVT::i1, MachineNode->getOperand(0), |
| 6864 |
MachineNode->getOperand(1). |
6864 |
MachineNode->getOperand(1). |
| 6865 |
getOperand(0)); |
6865 |
getOperand(0)); |
| 6866 |
else if (AllUsersSelectZero(MachineNode)) { |
6866 |
else if (AllUsersSelectZero(MachineNode)) { |
| 6867 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6867 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
| 6868 |
MVT::i1, MachineNode->getOperand(0), |
6868 |
MVT::i1, MachineNode->getOperand(0), |
| 6869 |
MachineNode->getOperand(1)); |
6869 |
MachineNode->getOperand(1)); |
| 6870 |
SelectSwap = true; |
6870 |
SelectSwap = true; |
| 6871 |
} |
6871 |
} |
| 6872 |
break; |
6872 |
break; |
| 6873 |
case PPC::CRXOR: |
6873 |
case PPC::CRXOR: |
| 6874 |
if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) |
6874 |
if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) |
| 6875 |
// xor(x, x) = 0 |
6875 |
// xor(x, x) = 0 |
| 6876 |
ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
6876 |
ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
| 6877 |
MVT::i1); |
6877 |
MVT::i1); |
| 6878 |
else if (Op1Set) |
6878 |
else if (Op1Set) |
| 6879 |
// xor(1, y) -> nor(y, y) |
6879 |
// xor(1, y) -> nor(y, y) |
| 6880 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6880 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
| 6881 |
MVT::i1, MachineNode->getOperand(1), |
6881 |
MVT::i1, MachineNode->getOperand(1), |
| 6882 |
MachineNode->getOperand(1)); |
6882 |
MachineNode->getOperand(1)); |
| 6883 |
else if (Op2Set) |
6883 |
else if (Op2Set) |
| 6884 |
// xor(x, 1) -> nor(x, x) |
6884 |
// xor(x, 1) -> nor(x, x) |
| 6885 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6885 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
| 6886 |
MVT::i1, MachineNode->getOperand(0), |
6886 |
MVT::i1, MachineNode->getOperand(0), |
| 6887 |
MachineNode->getOperand(0)); |
6887 |
MachineNode->getOperand(0)); |
| 6888 |
else if (Op1Unset) |
6888 |
else if (Op1Unset) |
| 6889 |
// xor(0, y) = y |
6889 |
// xor(0, y) = y |
| 6890 |
ResNode = MachineNode->getOperand(1).getNode(); |
6890 |
ResNode = MachineNode->getOperand(1).getNode(); |
| 6891 |
else if (Op2Unset) |
6891 |
else if (Op2Unset) |
| 6892 |
// xor(x, 0) = x |
6892 |
// xor(x, 0) = x |
| 6893 |
ResNode = MachineNode->getOperand(0).getNode(); |
6893 |
ResNode = MachineNode->getOperand(0).getNode(); |
| 6894 |
else if (Op1Not) |
6894 |
else if (Op1Not) |
| 6895 |
// xor(~x, y) = eqv(x, y) |
6895 |
// xor(~x, y) = eqv(x, y) |
| 6896 |
ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), |
6896 |
ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), |
| 6897 |
MVT::i1, MachineNode->getOperand(0). |
6897 |
MVT::i1, MachineNode->getOperand(0). |
| 6898 |
getOperand(0), |
6898 |
getOperand(0), |
| 6899 |
MachineNode->getOperand(1)); |
6899 |
MachineNode->getOperand(1)); |
| 6900 |
else if (Op2Not) |
6900 |
else if (Op2Not) |
| 6901 |
// xor(x, ~y) = eqv(x, y) |
6901 |
// xor(x, ~y) = eqv(x, y) |
| 6902 |
ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), |
6902 |
ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), |
| 6903 |
MVT::i1, MachineNode->getOperand(0), |
6903 |
MVT::i1, MachineNode->getOperand(0), |
| 6904 |
MachineNode->getOperand(1). |
6904 |
MachineNode->getOperand(1). |
| 6905 |
getOperand(0)); |
6905 |
getOperand(0)); |
| 6906 |
else if (AllUsersSelectZero(MachineNode)) { |
6906 |
else if (AllUsersSelectZero(MachineNode)) { |
| 6907 |
ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), |
6907 |
ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), |
| 6908 |
MVT::i1, MachineNode->getOperand(0), |
6908 |
MVT::i1, MachineNode->getOperand(0), |
| 6909 |
MachineNode->getOperand(1)); |
6909 |
MachineNode->getOperand(1)); |
| 6910 |
SelectSwap = true; |
6910 |
SelectSwap = true; |
| 6911 |
} |
6911 |
} |
| 6912 |
break; |
6912 |
break; |
| 6913 |
case PPC::CRNOR: |
6913 |
case PPC::CRNOR: |
| 6914 |
if (Op1Set || Op2Set) |
6914 |
if (Op1Set || Op2Set) |
| 6915 |
// nor(1, y) -> 0 |
6915 |
// nor(1, y) -> 0 |
| 6916 |
ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
6916 |
ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
| 6917 |
MVT::i1); |
6917 |
MVT::i1); |
| 6918 |
else if (Op1Unset) |
6918 |
else if (Op1Unset) |
| 6919 |
// nor(0, y) = ~y -> nor(y, y) |
6919 |
// nor(0, y) = ~y -> nor(y, y) |
| 6920 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6920 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
| 6921 |
MVT::i1, MachineNode->getOperand(1), |
6921 |
MVT::i1, MachineNode->getOperand(1), |
| 6922 |
MachineNode->getOperand(1)); |
6922 |
MachineNode->getOperand(1)); |
| 6923 |
else if (Op2Unset) |
6923 |
else if (Op2Unset) |
| 6924 |
// nor(x, 0) = ~x |
6924 |
// nor(x, 0) = ~x |
| 6925 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6925 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
| 6926 |
MVT::i1, MachineNode->getOperand(0), |
6926 |
MVT::i1, MachineNode->getOperand(0), |
| 6927 |
MachineNode->getOperand(0)); |
6927 |
MachineNode->getOperand(0)); |
| 6928 |
else if (Op1Not) |
6928 |
else if (Op1Not) |
| 6929 |
// nor(~x, y) = andc(x, y) |
6929 |
// nor(~x, y) = andc(x, y) |
| 6930 |
ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
6930 |
ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
| 6931 |
MVT::i1, MachineNode->getOperand(0). |
6931 |
MVT::i1, MachineNode->getOperand(0). |
| 6932 |
getOperand(0), |
6932 |
getOperand(0), |
| 6933 |
MachineNode->getOperand(1)); |
6933 |
MachineNode->getOperand(1)); |
| 6934 |
else if (Op2Not) |
6934 |
else if (Op2Not) |
| 6935 |
// nor(x, ~y) = andc(y, x) |
6935 |
// nor(x, ~y) = andc(y, x) |
| 6936 |
ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
6936 |
ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
| 6937 |
MVT::i1, MachineNode->getOperand(1). |
6937 |
MVT::i1, MachineNode->getOperand(1). |
| 6938 |
getOperand(0), |
6938 |
getOperand(0), |
| 6939 |
MachineNode->getOperand(0)); |
6939 |
MachineNode->getOperand(0)); |
| 6940 |
else if (AllUsersSelectZero(MachineNode)) { |
6940 |
else if (AllUsersSelectZero(MachineNode)) { |
| 6941 |
ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode), |
6941 |
ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode), |
| 6942 |
MVT::i1, MachineNode->getOperand(0), |
6942 |
MVT::i1, MachineNode->getOperand(0), |
| 6943 |
MachineNode->getOperand(1)); |
6943 |
MachineNode->getOperand(1)); |
| 6944 |
SelectSwap = true; |
6944 |
SelectSwap = true; |
| 6945 |
} |
6945 |
} |
| 6946 |
break; |
6946 |
break; |
| 6947 |
case PPC::CREQV: |
6947 |
case PPC::CREQV: |
| 6948 |
if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) |
6948 |
if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) |
| 6949 |
// eqv(x, x) = 1 |
6949 |
// eqv(x, x) = 1 |
| 6950 |
ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
6950 |
ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
| 6951 |
MVT::i1); |
6951 |
MVT::i1); |
| 6952 |
else if (Op1Set) |
6952 |
else if (Op1Set) |
| 6953 |
// eqv(1, y) = y |
6953 |
// eqv(1, y) = y |
| 6954 |
ResNode = MachineNode->getOperand(1).getNode(); |
6954 |
ResNode = MachineNode->getOperand(1).getNode(); |
| 6955 |
else if (Op2Set) |
6955 |
else if (Op2Set) |
| 6956 |
// eqv(x, 1) = x |
6956 |
// eqv(x, 1) = x |
| 6957 |
ResNode = MachineNode->getOperand(0).getNode(); |
6957 |
ResNode = MachineNode->getOperand(0).getNode(); |
| 6958 |
else if (Op1Unset) |
6958 |
else if (Op1Unset) |
| 6959 |
// eqv(0, y) = ~y -> nor(y, y) |
6959 |
// eqv(0, y) = ~y -> nor(y, y) |
| 6960 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6960 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
| 6961 |
MVT::i1, MachineNode->getOperand(1), |
6961 |
MVT::i1, MachineNode->getOperand(1), |
| 6962 |
MachineNode->getOperand(1)); |
6962 |
MachineNode->getOperand(1)); |
| 6963 |
else if (Op2Unset) |
6963 |
else if (Op2Unset) |
| 6964 |
// eqv(x, 0) = ~x |
6964 |
// eqv(x, 0) = ~x |
| 6965 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6965 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
| 6966 |
MVT::i1, MachineNode->getOperand(0), |
6966 |
MVT::i1, MachineNode->getOperand(0), |
| 6967 |
MachineNode->getOperand(0)); |
6967 |
MachineNode->getOperand(0)); |
| 6968 |
else if (Op1Not) |
6968 |
else if (Op1Not) |
| 6969 |
// eqv(~x, y) = xor(x, y) |
6969 |
// eqv(~x, y) = xor(x, y) |
| 6970 |
ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), |
6970 |
ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), |
| 6971 |
MVT::i1, MachineNode->getOperand(0). |
6971 |
MVT::i1, MachineNode->getOperand(0). |
| 6972 |
getOperand(0), |
6972 |
getOperand(0), |
| 6973 |
MachineNode->getOperand(1)); |
6973 |
MachineNode->getOperand(1)); |
| 6974 |
else if (Op2Not) |
6974 |
else if (Op2Not) |
| 6975 |
// eqv(x, ~y) = xor(x, y) |
6975 |
// eqv(x, ~y) = xor(x, y) |
| 6976 |
ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), |
6976 |
ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), |
| 6977 |
MVT::i1, MachineNode->getOperand(0), |
6977 |
MVT::i1, MachineNode->getOperand(0), |
| 6978 |
MachineNode->getOperand(1). |
6978 |
MachineNode->getOperand(1). |
| 6979 |
getOperand(0)); |
6979 |
getOperand(0)); |
| 6980 |
else if (AllUsersSelectZero(MachineNode)) { |
6980 |
else if (AllUsersSelectZero(MachineNode)) { |
| 6981 |
ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), |
6981 |
ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), |
| 6982 |
MVT::i1, MachineNode->getOperand(0), |
6982 |
MVT::i1, MachineNode->getOperand(0), |
| 6983 |
MachineNode->getOperand(1)); |
6983 |
MachineNode->getOperand(1)); |
| 6984 |
SelectSwap = true; |
6984 |
SelectSwap = true; |
| 6985 |
} |
6985 |
} |
| 6986 |
break; |
6986 |
break; |
| 6987 |
case PPC::CRANDC: |
6987 |
case PPC::CRANDC: |
| 6988 |
if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) |
6988 |
if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) |
| 6989 |
// andc(x, x) = 0 |
6989 |
// andc(x, x) = 0 |
| 6990 |
ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
6990 |
ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
| 6991 |
MVT::i1); |
6991 |
MVT::i1); |
| 6992 |
else if (Op1Set) |
6992 |
else if (Op1Set) |
| 6993 |
// andc(1, y) = ~y |
6993 |
// andc(1, y) = ~y |
| 6994 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6994 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
| 6995 |
MVT::i1, MachineNode->getOperand(1), |
6995 |
MVT::i1, MachineNode->getOperand(1), |
| 6996 |
MachineNode->getOperand(1)); |
6996 |
MachineNode->getOperand(1)); |
| 6997 |
else if (Op1Unset || Op2Set) |
6997 |
else if (Op1Unset || Op2Set) |
| 6998 |
// andc(0, y) = andc(x, 1) = 0 |
6998 |
// andc(0, y) = andc(x, 1) = 0 |
| 6999 |
ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
6999 |
ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
| 7000 |
MVT::i1); |
7000 |
MVT::i1); |
| 7001 |
else if (Op2Unset) |
7001 |
else if (Op2Unset) |
| 7002 |
// andc(x, 0) = x |
7002 |
// andc(x, 0) = x |
| 7003 |
ResNode = MachineNode->getOperand(0).getNode(); |
7003 |
ResNode = MachineNode->getOperand(0).getNode(); |
| 7004 |
else if (Op1Not) |
7004 |
else if (Op1Not) |
| 7005 |
// andc(~x, y) = ~(x | y) = nor(x, y) |
7005 |
// andc(~x, y) = ~(x | y) = nor(x, y) |
| 7006 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
7006 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
| 7007 |
MVT::i1, MachineNode->getOperand(0). |
7007 |
MVT::i1, MachineNode->getOperand(0). |
| 7008 |
getOperand(0), |
7008 |
getOperand(0), |
| 7009 |
MachineNode->getOperand(1)); |
7009 |
MachineNode->getOperand(1)); |
| 7010 |
else if (Op2Not) |
7010 |
else if (Op2Not) |
| 7011 |
// andc(x, ~y) = x & y |
7011 |
// andc(x, ~y) = x & y |
| 7012 |
ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode), |
7012 |
ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode), |
| 7013 |
MVT::i1, MachineNode->getOperand(0), |
7013 |
MVT::i1, MachineNode->getOperand(0), |
| 7014 |
MachineNode->getOperand(1). |
7014 |
MachineNode->getOperand(1). |
| 7015 |
getOperand(0)); |
7015 |
getOperand(0)); |
| 7016 |
else if (AllUsersSelectZero(MachineNode)) { |
7016 |
else if (AllUsersSelectZero(MachineNode)) { |
| 7017 |
ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
7017 |
ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
| 7018 |
MVT::i1, MachineNode->getOperand(1), |
7018 |
MVT::i1, MachineNode->getOperand(1), |
| 7019 |
MachineNode->getOperand(0)); |
7019 |
MachineNode->getOperand(0)); |
| 7020 |
SelectSwap = true; |
7020 |
SelectSwap = true; |
| 7021 |
} |
7021 |
} |
| 7022 |
break; |
7022 |
break; |
| 7023 |
case PPC::CRORC: |
7023 |
case PPC::CRORC: |
| 7024 |
if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) |
7024 |
if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) |
| 7025 |
// orc(x, x) = 1 |
7025 |
// orc(x, x) = 1 |
| 7026 |
ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
7026 |
ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
| 7027 |
MVT::i1); |
7027 |
MVT::i1); |
| 7028 |
else if (Op1Set || Op2Unset) |
7028 |
else if (Op1Set || Op2Unset) |
| 7029 |
// orc(1, y) = orc(x, 0) = 1 |
7029 |
// orc(1, y) = orc(x, 0) = 1 |
| 7030 |
ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
7030 |
ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
| 7031 |
MVT::i1); |
7031 |
MVT::i1); |
| 7032 |
else if (Op2Set) |
7032 |
else if (Op2Set) |
| 7033 |
// orc(x, 1) = x |
7033 |
// orc(x, 1) = x |
| 7034 |
ResNode = MachineNode->getOperand(0).getNode(); |
7034 |
ResNode = MachineNode->getOperand(0).getNode(); |
| 7035 |
else if (Op1Unset) |
7035 |
else if (Op1Unset) |
| 7036 |
// orc(0, y) = ~y |
7036 |
// orc(0, y) = ~y |
| 7037 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
7037 |
ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
| 7038 |
MVT::i1, MachineNode->getOperand(1), |
7038 |
MVT::i1, MachineNode->getOperand(1), |
| 7039 |
MachineNode->getOperand(1)); |
7039 |
MachineNode->getOperand(1)); |
| 7040 |
else if (Op1Not) |
7040 |
else if (Op1Not) |
| 7041 |
// orc(~x, y) = ~(x & y) = nand(x, y) |
7041 |
// orc(~x, y) = ~(x & y) = nand(x, y) |
| 7042 |
ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode), |
7042 |
ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode), |
| 7043 |
MVT::i1, MachineNode->getOperand(0). |
7043 |
MVT::i1, MachineNode->getOperand(0). |
| 7044 |
getOperand(0), |
7044 |
getOperand(0), |
| 7045 |
MachineNode->getOperand(1)); |
7045 |
MachineNode->getOperand(1)); |
| 7046 |
else if (Op2Not) |
7046 |
else if (Op2Not) |
| 7047 |
// orc(x, ~y) = x | y |
7047 |
// orc(x, ~y) = x | y |
| 7048 |
ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode), |
7048 |
ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode), |
| 7049 |
MVT::i1, MachineNode->getOperand(0), |
7049 |
MVT::i1, MachineNode->getOperand(0), |
| 7050 |
MachineNode->getOperand(1). |
7050 |
MachineNode->getOperand(1). |
| 7051 |
getOperand(0)); |
7051 |
getOperand(0)); |
| 7052 |
else if (AllUsersSelectZero(MachineNode)) { |
7052 |
else if (AllUsersSelectZero(MachineNode)) { |
| 7053 |
ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
7053 |
ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
| 7054 |
MVT::i1, MachineNode->getOperand(1), |
7054 |
MVT::i1, MachineNode->getOperand(1), |
| 7055 |
MachineNode->getOperand(0)); |
7055 |
MachineNode->getOperand(0)); |
| 7056 |
SelectSwap = true; |
7056 |
SelectSwap = true; |
| 7057 |
} |
7057 |
} |
| 7058 |
break; |
7058 |
break; |
| 7059 |
case PPC::SELECT_I4: |
7059 |
case PPC::SELECT_I4: |
| 7060 |
case PPC::SELECT_I8: |
7060 |
case PPC::SELECT_I8: |
| 7061 |
case PPC::SELECT_F4: |
7061 |
case PPC::SELECT_F4: |
| 7062 |
case PPC::SELECT_F8: |
7062 |
case PPC::SELECT_F8: |
| 7063 |
case PPC::SELECT_SPE: |
7063 |
case PPC::SELECT_SPE: |
| 7064 |
case PPC::SELECT_SPE4: |
7064 |
case PPC::SELECT_SPE4: |
| 7065 |
case PPC::SELECT_VRRC: |
7065 |
case PPC::SELECT_VRRC: |
| 7066 |
case PPC::SELECT_VSFRC: |
7066 |
case PPC::SELECT_VSFRC: |
| 7067 |
case PPC::SELECT_VSSRC: |
7067 |
case PPC::SELECT_VSSRC: |
| 7068 |
case PPC::SELECT_VSRC: |
7068 |
case PPC::SELECT_VSRC: |
| 7069 |
if (Op1Set) |
7069 |
if (Op1Set) |
| 7070 |
ResNode = MachineNode->getOperand(1).getNode(); |
7070 |
ResNode = MachineNode->getOperand(1).getNode(); |
| 7071 |
else if (Op1Unset) |
7071 |
else if (Op1Unset) |
| 7072 |
ResNode = MachineNode->getOperand(2).getNode(); |
7072 |
ResNode = MachineNode->getOperand(2).getNode(); |
| 7073 |
else if (Op1Not) |
7073 |
else if (Op1Not) |
| 7074 |
ResNode = CurDAG->getMachineNode(MachineNode->getMachineOpcode(), |
7074 |
ResNode = CurDAG->getMachineNode(MachineNode->getMachineOpcode(), |
| 7075 |
SDLoc(MachineNode), |
7075 |
SDLoc(MachineNode), |
| 7076 |
MachineNode->getValueType(0), |
7076 |
MachineNode->getValueType(0), |
| 7077 |
MachineNode->getOperand(0). |
7077 |
MachineNode->getOperand(0). |
| 7078 |
getOperand(0), |
7078 |
getOperand(0), |
| 7079 |
MachineNode->getOperand(2), |
7079 |
MachineNode->getOperand(2), |
| 7080 |
MachineNode->getOperand(1)); |
7080 |
MachineNode->getOperand(1)); |
| 7081 |
break; |
7081 |
break; |
| 7082 |
case PPC::BC: |
7082 |
case PPC::BC: |
| 7083 |
case PPC::BCn: |
7083 |
case PPC::BCn: |
| 7084 |
if (Op1Not) |
7084 |
if (Op1Not) |
| 7085 |
ResNode = CurDAG->getMachineNode(Opcode == PPC::BC ? PPC::BCn : |
7085 |
ResNode = CurDAG->getMachineNode(Opcode == PPC::BC ? PPC::BCn : |
| 7086 |
PPC::BC, |
7086 |
PPC::BC, |
| 7087 |
SDLoc(MachineNode), |
7087 |
SDLoc(MachineNode), |
| 7088 |
MVT::Other, |
7088 |
MVT::Other, |
| 7089 |
MachineNode->getOperand(0). |
7089 |
MachineNode->getOperand(0). |
| 7090 |
getOperand(0), |
7090 |
getOperand(0), |
| 7091 |
MachineNode->getOperand(1), |
7091 |
MachineNode->getOperand(1), |
| 7092 |
MachineNode->getOperand(2)); |
7092 |
MachineNode->getOperand(2)); |
| 7093 |
// FIXME: Handle Op1Set, Op1Unset here too. |
7093 |
// FIXME: Handle Op1Set, Op1Unset here too. |
| 7094 |
break; |
7094 |
break; |
| 7095 |
} |
7095 |
} |
| 7096 |
|
7096 |
|
| 7097 |
// If we're inverting this node because it is used only by selects that |
7097 |
// If we're inverting this node because it is used only by selects that |
| 7098 |
// we'd like to swap, then swap the selects before the node replacement. |
7098 |
// we'd like to swap, then swap the selects before the node replacement. |
| 7099 |
if (SelectSwap) |
7099 |
if (SelectSwap) |
| 7100 |
SwapAllSelectUsers(MachineNode); |
7100 |
SwapAllSelectUsers(MachineNode); |
| 7101 |
|
7101 |
|
| 7102 |
if (ResNode != MachineNode) { |
7102 |
if (ResNode != MachineNode) { |
| 7103 |
LLVM_DEBUG(dbgs() << "CR Peephole replacing:\nOld: "); |
7103 |
LLVM_DEBUG(dbgs() << "CR Peephole replacing:\nOld: "); |
| 7104 |
LLVM_DEBUG(MachineNode->dump(CurDAG)); |
7104 |
LLVM_DEBUG(MachineNode->dump(CurDAG)); |
| 7105 |
LLVM_DEBUG(dbgs() << "\nNew: "); |
7105 |
LLVM_DEBUG(dbgs() << "\nNew: "); |
| 7106 |
LLVM_DEBUG(ResNode->dump(CurDAG)); |
7106 |
LLVM_DEBUG(ResNode->dump(CurDAG)); |
| 7107 |
LLVM_DEBUG(dbgs() << "\n"); |
7107 |
LLVM_DEBUG(dbgs() << "\n"); |
| 7108 |
|
7108 |
|
| 7109 |
ReplaceUses(MachineNode, ResNode); |
7109 |
ReplaceUses(MachineNode, ResNode); |
| 7110 |
IsModified = true; |
7110 |
IsModified = true; |
| 7111 |
} |
7111 |
} |
| 7112 |
} |
7112 |
} |
| 7113 |
if (IsModified) |
7113 |
if (IsModified) |
| 7114 |
CurDAG->RemoveDeadNodes(); |
7114 |
CurDAG->RemoveDeadNodes(); |
| 7115 |
} while (IsModified); |
7115 |
} while (IsModified); |
| 7116 |
} |
7116 |
} |
| 7117 |
|
7117 |
|
| 7118 |
// Gather the set of 32-bit operations that are known to have their |
7118 |
// Gather the set of 32-bit operations that are known to have their |
| 7119 |
// higher-order 32 bits zero, where ToPromote contains all such operations. |
7119 |
// higher-order 32 bits zero, where ToPromote contains all such operations. |
| 7120 |
static bool PeepholePPC64ZExtGather(SDValue Op32, |
7120 |
static bool PeepholePPC64ZExtGather(SDValue Op32, |
| 7121 |
SmallPtrSetImpl &ToPromote) { |
7121 |
SmallPtrSetImpl &ToPromote) { |
| 7122 |
if (!Op32.isMachineOpcode()) |
7122 |
if (!Op32.isMachineOpcode()) |
| 7123 |
return false; |
7123 |
return false; |
| 7124 |
|
7124 |
|
| 7125 |
// First, check for the "frontier" instructions (those that will clear the |
7125 |
// First, check for the "frontier" instructions (those that will clear the |
| 7126 |
// higher-order 32 bits. |
7126 |
// higher-order 32 bits. |
| 7127 |
|
7127 |
|
| 7128 |
// For RLWINM and RLWNM, we need to make sure that the mask does not wrap |
7128 |
// For RLWINM and RLWNM, we need to make sure that the mask does not wrap |
| 7129 |
// around. If it does not, then these instructions will clear the |
7129 |
// around. If it does not, then these instructions will clear the |
| 7130 |
// higher-order bits. |
7130 |
// higher-order bits. |
| 7131 |
if ((Op32.getMachineOpcode() == PPC::RLWINM || |
7131 |
if ((Op32.getMachineOpcode() == PPC::RLWINM || |
| 7132 |
Op32.getMachineOpcode() == PPC::RLWNM) && |
7132 |
Op32.getMachineOpcode() == PPC::RLWNM) && |
| 7133 |
Op32.getConstantOperandVal(2) <= Op32.getConstantOperandVal(3)) { |
7133 |
Op32.getConstantOperandVal(2) <= Op32.getConstantOperandVal(3)) { |
| 7134 |
ToPromote.insert(Op32.getNode()); |
7134 |
ToPromote.insert(Op32.getNode()); |
| 7135 |
return true; |
7135 |
return true; |
| 7136 |
} |
7136 |
} |
| 7137 |
|
7137 |
|
| 7138 |
// SLW and SRW always clear the higher-order bits. |
7138 |
// SLW and SRW always clear the higher-order bits. |
| 7139 |
if (Op32.getMachineOpcode() == PPC::SLW || |
7139 |
if (Op32.getMachineOpcode() == PPC::SLW || |
| 7140 |
Op32.getMachineOpcode() == PPC::SRW) { |
7140 |
Op32.getMachineOpcode() == PPC::SRW) { |
| 7141 |
ToPromote.insert(Op32.getNode()); |
7141 |
ToPromote.insert(Op32.getNode()); |
| 7142 |
return true; |
7142 |
return true; |
| 7143 |
} |
7143 |
} |
| 7144 |
|
7144 |
|
| 7145 |
// For LI and LIS, we need the immediate to be positive (so that it is not |
7145 |
// For LI and LIS, we need the immediate to be positive (so that it is not |
| 7146 |
// sign extended). |
7146 |
// sign extended). |
| 7147 |
if (Op32.getMachineOpcode() == PPC::LI || |
7147 |
if (Op32.getMachineOpcode() == PPC::LI || |
| 7148 |
Op32.getMachineOpcode() == PPC::LIS) { |
7148 |
Op32.getMachineOpcode() == PPC::LIS) { |
| 7149 |
if (!isUInt<15>(Op32.getConstantOperandVal(0))) |
7149 |
if (!isUInt<15>(Op32.getConstantOperandVal(0))) |
| 7150 |
return false; |
7150 |
return false; |
| 7151 |
|
7151 |
|
| 7152 |
ToPromote.insert(Op32.getNode()); |
7152 |
ToPromote.insert(Op32.getNode()); |
| 7153 |
return true; |
7153 |
return true; |
| 7154 |
} |
7154 |
} |
| 7155 |
|
7155 |
|
| 7156 |
// LHBRX and LWBRX always clear the higher-order bits. |
7156 |
// LHBRX and LWBRX always clear the higher-order bits. |
| 7157 |
if (Op32.getMachineOpcode() == PPC::LHBRX || |
7157 |
if (Op32.getMachineOpcode() == PPC::LHBRX || |
| 7158 |
Op32.getMachineOpcode() == PPC::LWBRX) { |
7158 |
Op32.getMachineOpcode() == PPC::LWBRX) { |
| 7159 |
ToPromote.insert(Op32.getNode()); |
7159 |
ToPromote.insert(Op32.getNode()); |
| 7160 |
return true; |
7160 |
return true; |
| 7161 |
} |
7161 |
} |
| 7162 |
|
7162 |
|
| 7163 |
// CNT[LT]ZW always produce a 64-bit value in [0,32], and so is zero extended. |
7163 |
// CNT[LT]ZW always produce a 64-bit value in [0,32], and so is zero extended. |
| 7164 |
if (Op32.getMachineOpcode() == PPC::CNTLZW || |
7164 |
if (Op32.getMachineOpcode() == PPC::CNTLZW || |
| 7165 |
Op32.getMachineOpcode() == PPC::CNTTZW) { |
7165 |
Op32.getMachineOpcode() == PPC::CNTTZW) { |
| 7166 |
ToPromote.insert(Op32.getNode()); |
7166 |
ToPromote.insert(Op32.getNode()); |
| 7167 |
return true; |
7167 |
return true; |
| 7168 |
} |
7168 |
} |
| 7169 |
|
7169 |
|
| 7170 |
// Next, check for those instructions we can look through. |
7170 |
// Next, check for those instructions we can look through. |
| 7171 |
|
7171 |
|
| 7172 |
// Assuming the mask does not wrap around, then the higher-order bits are |
7172 |
// Assuming the mask does not wrap around, then the higher-order bits are |
| 7173 |
// taken directly from the first operand. |
7173 |
// taken directly from the first operand. |
| 7174 |
if (Op32.getMachineOpcode() == PPC::RLWIMI && |
7174 |
if (Op32.getMachineOpcode() == PPC::RLWIMI && |
| 7175 |
Op32.getConstantOperandVal(3) <= Op32.getConstantOperandVal(4)) { |
7175 |
Op32.getConstantOperandVal(3) <= Op32.getConstantOperandVal(4)) { |
| 7176 |
SmallPtrSet ToPromote1; |
7176 |
SmallPtrSet ToPromote1; |
| 7177 |
if (!PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1)) |
7177 |
if (!PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1)) |
| 7178 |
return false; |
7178 |
return false; |
| 7179 |
|
7179 |
|
| 7180 |
ToPromote.insert(Op32.getNode()); |
7180 |
ToPromote.insert(Op32.getNode()); |
| 7181 |
ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); |
7181 |
ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); |
| 7182 |
return true; |
7182 |
return true; |
| 7183 |
} |
7183 |
} |
| 7184 |
|
7184 |
|
| 7185 |
// For OR, the higher-order bits are zero if that is true for both operands. |
7185 |
// For OR, the higher-order bits are zero if that is true for both operands. |
| 7186 |
// For SELECT_I4, the same is true (but the relevant operand numbers are |
7186 |
// For SELECT_I4, the same is true (but the relevant operand numbers are |
| 7187 |
// shifted by 1). |
7187 |
// shifted by 1). |
| 7188 |
if (Op32.getMachineOpcode() == PPC::OR || |
7188 |
if (Op32.getMachineOpcode() == PPC::OR || |
| 7189 |
Op32.getMachineOpcode() == PPC::SELECT_I4) { |
7189 |
Op32.getMachineOpcode() == PPC::SELECT_I4) { |
| 7190 |
unsigned B = Op32.getMachineOpcode() == PPC::SELECT_I4 ? 1 : 0; |
7190 |
unsigned B = Op32.getMachineOpcode() == PPC::SELECT_I4 ? 1 : 0; |
| 7191 |
SmallPtrSet ToPromote1; |
7191 |
SmallPtrSet ToPromote1; |
| 7192 |
if (!PeepholePPC64ZExtGather(Op32.getOperand(B+0), ToPromote1)) |
7192 |
if (!PeepholePPC64ZExtGather(Op32.getOperand(B+0), ToPromote1)) |
| 7193 |
return false; |
7193 |
return false; |
| 7194 |
if (!PeepholePPC64ZExtGather(Op32.getOperand(B+1), ToPromote1)) |
7194 |
if (!PeepholePPC64ZExtGather(Op32.getOperand(B+1), ToPromote1)) |
| 7195 |
return false; |
7195 |
return false; |
| 7196 |
|
7196 |
|
| 7197 |
ToPromote.insert(Op32.getNode()); |
7197 |
ToPromote.insert(Op32.getNode()); |
| 7198 |
ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); |
7198 |
ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); |
| 7199 |
return true; |
7199 |
return true; |
| 7200 |
} |
7200 |
} |
| 7201 |
|
7201 |
|
| 7202 |
// For ORI and ORIS, we need the higher-order bits of the first operand to be |
7202 |
// For ORI and ORIS, we need the higher-order bits of the first operand to be |
| 7203 |
// zero, and also for the constant to be positive (so that it is not sign |
7203 |
// zero, and also for the constant to be positive (so that it is not sign |
| 7204 |
// extended). |
7204 |
// extended). |
| 7205 |
if (Op32.getMachineOpcode() == PPC::ORI || |
7205 |
if (Op32.getMachineOpcode() == PPC::ORI || |
| 7206 |
Op32.getMachineOpcode() == PPC::ORIS) { |
7206 |
Op32.getMachineOpcode() == PPC::ORIS) { |
| 7207 |
SmallPtrSet ToPromote1; |
7207 |
SmallPtrSet ToPromote1; |
| 7208 |
if (!PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1)) |
7208 |
if (!PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1)) |
| 7209 |
return false; |
7209 |
return false; |
| 7210 |
if (!isUInt<15>(Op32.getConstantOperandVal(1))) |
7210 |
if (!isUInt<15>(Op32.getConstantOperandVal(1))) |
| 7211 |
return false; |
7211 |
return false; |
| 7212 |
|
7212 |
|
| 7213 |
ToPromote.insert(Op32.getNode()); |
7213 |
ToPromote.insert(Op32.getNode()); |
| 7214 |
ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); |
7214 |
ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); |
| 7215 |
return true; |
7215 |
return true; |
| 7216 |
} |
7216 |
} |
| 7217 |
|
7217 |
|
| 7218 |
// The higher-order bits of AND are zero if that is true for at least one of |
7218 |
// The higher-order bits of AND are zero if that is true for at least one of |
| 7219 |
// the operands. |
7219 |
// the operands. |
| 7220 |
if (Op32.getMachineOpcode() == PPC::AND) { |
7220 |
if (Op32.getMachineOpcode() == PPC::AND) { |
| 7221 |
SmallPtrSet ToPromote1, ToPromote2; |
7221 |
SmallPtrSet ToPromote1, ToPromote2; |
| 7222 |
bool Op0OK = |
7222 |
bool Op0OK = |
| 7223 |
PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1); |
7223 |
PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1); |
| 7224 |
bool Op1OK = |
7224 |
bool Op1OK = |
| 7225 |
PeepholePPC64ZExtGather(Op32.getOperand(1), ToPromote2); |
7225 |
PeepholePPC64ZExtGather(Op32.getOperand(1), ToPromote2); |
| 7226 |
if (!Op0OK && !Op1OK) |
7226 |
if (!Op0OK && !Op1OK) |
| 7227 |
return false; |
7227 |
return false; |
| 7228 |
|
7228 |
|
| 7229 |
ToPromote.insert(Op32.getNode()); |
7229 |
ToPromote.insert(Op32.getNode()); |
| 7230 |
|
7230 |
|
| 7231 |
if (Op0OK) |
7231 |
if (Op0OK) |
| 7232 |
ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); |
7232 |
ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); |
| 7233 |
|
7233 |
|
| 7234 |
if (Op1OK) |
7234 |
if (Op1OK) |
| 7235 |
ToPromote.insert(ToPromote2.begin(), ToPromote2.end()); |
7235 |
ToPromote.insert(ToPromote2.begin(), ToPromote2.end()); |
| 7236 |
|
7236 |
|
| 7237 |
return true; |
7237 |
return true; |
| 7238 |
} |
7238 |
} |
| 7239 |
|
7239 |
|
| 7240 |
// For ANDI and ANDIS, the higher-order bits are zero if either that is true |
7240 |
// For ANDI and ANDIS, the higher-order bits are zero if either that is true |
| 7241 |
// of the first operand, or if the second operand is positive (so that it is |
7241 |
// of the first operand, or if the second operand is positive (so that it is |
| 7242 |
// not sign extended). |
7242 |
// not sign extended). |
| 7243 |
if (Op32.getMachineOpcode() == PPC::ANDI_rec || |
7243 |
if (Op32.getMachineOpcode() == PPC::ANDI_rec || |
| 7244 |
Op32.getMachineOpcode() == PPC::ANDIS_rec) { |
7244 |
Op32.getMachineOpcode() == PPC::ANDIS_rec) { |
| 7245 |
SmallPtrSet ToPromote1; |
7245 |
SmallPtrSet ToPromote1; |
| 7246 |
bool Op0OK = |
7246 |
bool Op0OK = |
| 7247 |
PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1); |
7247 |
PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1); |
| 7248 |
bool Op1OK = isUInt<15>(Op32.getConstantOperandVal(1)); |
7248 |
bool Op1OK = isUInt<15>(Op32.getConstantOperandVal(1)); |
| 7249 |
if (!Op0OK && !Op1OK) |
7249 |
if (!Op0OK && !Op1OK) |
| 7250 |
return false; |
7250 |
return false; |
| 7251 |
|
7251 |
|
| 7252 |
ToPromote.insert(Op32.getNode()); |
7252 |
ToPromote.insert(Op32.getNode()); |
| 7253 |
|
7253 |
|
| 7254 |
if (Op0OK) |
7254 |
if (Op0OK) |
| 7255 |
ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); |
7255 |
ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); |
| 7256 |
|
7256 |
|
| 7257 |
return true; |
7257 |
return true; |
| 7258 |
} |
7258 |
} |
| 7259 |
|
7259 |
|
| 7260 |
return false; |
7260 |
return false; |
| 7261 |
} |
7261 |
} |
| 7262 |
|
7262 |
|
| 7263 |
void PPCDAGToDAGISel::PeepholePPC64ZExt() { |
7263 |
void PPCDAGToDAGISel::PeepholePPC64ZExt() { |
| 7264 |
if (!Subtarget->isPPC64()) |
7264 |
if (!Subtarget->isPPC64()) |
| 7265 |
return; |
7265 |
return; |
| 7266 |
|
7266 |
|
| 7267 |
// When we zero-extend from i32 to i64, we use a pattern like this: |
7267 |
// When we zero-extend from i32 to i64, we use a pattern like this: |
| 7268 |
// def : Pat<(i64 (zext i32:$in)), |
7268 |
// def : Pat<(i64 (zext i32:$in)), |
| 7269 |
// (RLDICL (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $in, sub_32), |
7269 |
// (RLDICL (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $in, sub_32), |
| 7270 |
// 0, 32)>; |
7270 |
// 0, 32)>; |
| 7271 |
// There are several 32-bit shift/rotate instructions, however, that will |
7271 |
// There are several 32-bit shift/rotate instructions, however, that will |
| 7272 |
// clear the higher-order bits of their output, rendering the RLDICL |
7272 |
// clear the higher-order bits of their output, rendering the RLDICL |
| 7273 |
// unnecessary. When that happens, we remove it here, and redefine the |
7273 |
// unnecessary. When that happens, we remove it here, and redefine the |
| 7274 |
// relevant 32-bit operation to be a 64-bit operation. |
7274 |
// relevant 32-bit operation to be a 64-bit operation. |
| 7275 |
|
7275 |
|
| 7276 |
SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); |
7276 |
SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); |
| 7277 |
|
7277 |
|
| 7278 |
bool MadeChange = false; |
7278 |
bool MadeChange = false; |
| 7279 |
while (Position != CurDAG->allnodes_begin()) { |
7279 |
while (Position != CurDAG->allnodes_begin()) { |
| 7280 |
SDNode *N = &*--Position; |
7280 |
SDNode *N = &*--Position; |
| 7281 |
// Skip dead nodes and any non-machine opcodes. |
7281 |
// Skip dead nodes and any non-machine opcodes. |
| 7282 |
if (N->use_empty() || !N->isMachineOpcode()) |
7282 |
if (N->use_empty() || !N->isMachineOpcode()) |
| 7283 |
continue; |
7283 |
continue; |
| 7284 |
|
7284 |
|
| 7285 |
if (N->getMachineOpcode() != PPC::RLDICL) |
7285 |
if (N->getMachineOpcode() != PPC::RLDICL) |
| 7286 |
continue; |
7286 |
continue; |
| 7287 |
|
7287 |
|
| 7288 |
if (N->getConstantOperandVal(1) != 0 || |
7288 |
if (N->getConstantOperandVal(1) != 0 || |
| 7289 |
N->getConstantOperandVal(2) != 32) |
7289 |
N->getConstantOperandVal(2) != 32) |
| 7290 |
continue; |
7290 |
continue; |
| 7291 |
|
7291 |
|
| 7292 |
SDValue ISR = N->getOperand(0); |
7292 |
SDValue ISR = N->getOperand(0); |
| 7293 |
if (!ISR.isMachineOpcode() || |
7293 |
if (!ISR.isMachineOpcode() || |
| 7294 |
ISR.getMachineOpcode() != TargetOpcode::INSERT_SUBREG) |
7294 |
ISR.getMachineOpcode() != TargetOpcode::INSERT_SUBREG) |
| 7295 |
continue; |
7295 |
continue; |
| 7296 |
|
7296 |
|
| 7297 |
if (!ISR.hasOneUse()) |
7297 |
if (!ISR.hasOneUse()) |
| 7298 |
continue; |
7298 |
continue; |
| 7299 |
|
7299 |
|
| 7300 |
if (ISR.getConstantOperandVal(2) != PPC::sub_32) |
7300 |
if (ISR.getConstantOperandVal(2) != PPC::sub_32) |
| 7301 |
continue; |
7301 |
continue; |
| 7302 |
|
7302 |
|
| 7303 |
SDValue IDef = ISR.getOperand(0); |
7303 |
SDValue IDef = ISR.getOperand(0); |
| 7304 |
if (!IDef.isMachineOpcode() || |
7304 |
if (!IDef.isMachineOpcode() || |
| 7305 |
IDef.getMachineOpcode() != TargetOpcode::IMPLICIT_DEF) |
7305 |
IDef.getMachineOpcode() != TargetOpcode::IMPLICIT_DEF) |
| 7306 |
continue; |
7306 |
continue; |
| 7307 |
|
7307 |
|
| 7308 |
// We now know that we're looking at a canonical i32 -> i64 zext. See if we |
7308 |
// We now know that we're looking at a canonical i32 -> i64 zext. See if we |
| 7309 |
// can get rid of it. |
7309 |
// can get rid of it. |
| 7310 |
|
7310 |
|
| 7311 |
SDValue Op32 = ISR->getOperand(1); |
7311 |
SDValue Op32 = ISR->getOperand(1); |
| 7312 |
if (!Op32.isMachineOpcode()) |
7312 |
if (!Op32.isMachineOpcode()) |
| 7313 |
continue; |
7313 |
continue; |
| 7314 |
|
7314 |
|
| 7315 |
// There are some 32-bit instructions that always clear the high-order 32 |
7315 |
// There are some 32-bit instructions that always clear the high-order 32 |
| 7316 |
// bits, there are also some instructions (like AND) that we can look |
7316 |
// bits, there are also some instructions (like AND) that we can look |
| 7317 |
// through. |
7317 |
// through. |
| 7318 |
SmallPtrSet ToPromote; |
7318 |
SmallPtrSet ToPromote; |
| 7319 |
if (!PeepholePPC64ZExtGather(Op32, ToPromote)) |
7319 |
if (!PeepholePPC64ZExtGather(Op32, ToPromote)) |
| 7320 |
continue; |
7320 |
continue; |
| 7321 |
|
7321 |
|
| 7322 |
// If the ToPromote set contains nodes that have uses outside of the set |
7322 |
// If the ToPromote set contains nodes that have uses outside of the set |
| 7323 |
// (except for the original INSERT_SUBREG), then abort the transformation. |
7323 |
// (except for the original INSERT_SUBREG), then abort the transformation. |
| 7324 |
bool OutsideUse = false; |
7324 |
bool OutsideUse = false; |
| 7325 |
for (SDNode *PN : ToPromote) { |
7325 |
for (SDNode *PN : ToPromote) { |
| 7326 |
for (SDNode *UN : PN->uses()) { |
7326 |
for (SDNode *UN : PN->uses()) { |
| 7327 |
if (!ToPromote.count(UN) && UN != ISR.getNode()) { |
7327 |
if (!ToPromote.count(UN) && UN != ISR.getNode()) { |
| 7328 |
OutsideUse = true; |
7328 |
OutsideUse = true; |
| 7329 |
break; |
7329 |
break; |
| 7330 |
} |
7330 |
} |
| 7331 |
} |
7331 |
} |
| 7332 |
|
7332 |
|
| 7333 |
if (OutsideUse) |
7333 |
if (OutsideUse) |
| 7334 |
break; |
7334 |
break; |
| 7335 |
} |
7335 |
} |
| 7336 |
if (OutsideUse) |
7336 |
if (OutsideUse) |
| 7337 |
continue; |
7337 |
continue; |
| 7338 |
|
7338 |
|
| 7339 |
MadeChange = true; |
7339 |
MadeChange = true; |
| 7340 |
|
7340 |
|
| 7341 |
// We now know that this zero extension can be removed by promoting to |
7341 |
// We now know that this zero extension can be removed by promoting to |
| 7342 |
// nodes in ToPromote to 64-bit operations, where for operations in the |
7342 |
// nodes in ToPromote to 64-bit operations, where for operations in the |
| 7343 |
// frontier of the set, we need to insert INSERT_SUBREGs for their |
7343 |
// frontier of the set, we need to insert INSERT_SUBREGs for their |
| 7344 |
// operands. |
7344 |
// operands. |
| 7345 |
for (SDNode *PN : ToPromote) { |
7345 |
for (SDNode *PN : ToPromote) { |
| 7346 |
unsigned NewOpcode; |
7346 |
unsigned NewOpcode; |
| 7347 |
switch (PN->getMachineOpcode()) { |
7347 |
switch (PN->getMachineOpcode()) { |
| 7348 |
default: |
7348 |
default: |
| 7349 |
llvm_unreachable("Don't know the 64-bit variant of this instruction"); |
7349 |
llvm_unreachable("Don't know the 64-bit variant of this instruction"); |
| 7350 |
case PPC::RLWINM: NewOpcode = PPC::RLWINM8; break; |
7350 |
case PPC::RLWINM: NewOpcode = PPC::RLWINM8; break; |
| 7351 |
case PPC::RLWNM: NewOpcode = PPC::RLWNM8; break; |
7351 |
case PPC::RLWNM: NewOpcode = PPC::RLWNM8; break; |
| 7352 |
case PPC::SLW: NewOpcode = PPC::SLW8; break; |
7352 |
case PPC::SLW: NewOpcode = PPC::SLW8; break; |
| 7353 |
case PPC::SRW: NewOpcode = PPC::SRW8; break; |
7353 |
case PPC::SRW: NewOpcode = PPC::SRW8; break; |
| 7354 |
case PPC::LI: NewOpcode = PPC::LI8; break; |
7354 |
case PPC::LI: NewOpcode = PPC::LI8; break; |
| 7355 |
case PPC::LIS: NewOpcode = PPC::LIS8; break; |
7355 |
case PPC::LIS: NewOpcode = PPC::LIS8; break; |
| 7356 |
case PPC::LHBRX: NewOpcode = PPC::LHBRX8; break; |
7356 |
case PPC::LHBRX: NewOpcode = PPC::LHBRX8; break; |
| 7357 |
case PPC::LWBRX: NewOpcode = PPC::LWBRX8; break; |
7357 |
case PPC::LWBRX: NewOpcode = PPC::LWBRX8; break; |
| 7358 |
case PPC::CNTLZW: NewOpcode = PPC::CNTLZW8; break; |
7358 |
case PPC::CNTLZW: NewOpcode = PPC::CNTLZW8; break; |
| 7359 |
case PPC::CNTTZW: NewOpcode = PPC::CNTTZW8; break; |
7359 |
case PPC::CNTTZW: NewOpcode = PPC::CNTTZW8; break; |
| 7360 |
case PPC::RLWIMI: NewOpcode = PPC::RLWIMI8; break; |
7360 |
case PPC::RLWIMI: NewOpcode = PPC::RLWIMI8; break; |
| 7361 |
case PPC::OR: NewOpcode = PPC::OR8; break; |
7361 |
case PPC::OR: NewOpcode = PPC::OR8; break; |
| 7362 |
case PPC::SELECT_I4: NewOpcode = PPC::SELECT_I8; break; |
7362 |
case PPC::SELECT_I4: NewOpcode = PPC::SELECT_I8; break; |
| 7363 |
case PPC::ORI: NewOpcode = PPC::ORI8; break; |
7363 |
case PPC::ORI: NewOpcode = PPC::ORI8; break; |
| 7364 |
case PPC::ORIS: NewOpcode = PPC::ORIS8; break; |
7364 |
case PPC::ORIS: NewOpcode = PPC::ORIS8; break; |
| 7365 |
case PPC::AND: NewOpcode = PPC::AND8; break; |
7365 |
case PPC::AND: NewOpcode = PPC::AND8; break; |
| 7366 |
case PPC::ANDI_rec: |
7366 |
case PPC::ANDI_rec: |
| 7367 |
NewOpcode = PPC::ANDI8_rec; |
7367 |
NewOpcode = PPC::ANDI8_rec; |
| 7368 |
break; |
7368 |
break; |
| 7369 |
case PPC::ANDIS_rec: |
7369 |
case PPC::ANDIS_rec: |
| 7370 |
NewOpcode = PPC::ANDIS8_rec; |
7370 |
NewOpcode = PPC::ANDIS8_rec; |
| 7371 |
break; |
7371 |
break; |
| 7372 |
} |
7372 |
} |
| 7373 |
|
7373 |
|
| 7374 |
// Note: During the replacement process, the nodes will be in an |
7374 |
// Note: During the replacement process, the nodes will be in an |
| 7375 |
// inconsistent state (some instructions will have operands with values |
7375 |
// inconsistent state (some instructions will have operands with values |
| 7376 |
// of the wrong type). Once done, however, everything should be right |
7376 |
// of the wrong type). Once done, however, everything should be right |
| 7377 |
// again. |
7377 |
// again. |
| 7378 |
|
7378 |
|
| 7379 |
SmallVector Ops; |
7379 |
SmallVector Ops; |
| 7380 |
for (const SDValue &V : PN->ops()) { |
7380 |
for (const SDValue &V : PN->ops()) { |
| 7381 |
if (!ToPromote.count(V.getNode()) && V.getValueType() == MVT::i32 && |
7381 |
if (!ToPromote.count(V.getNode()) && V.getValueType() == MVT::i32 && |
| 7382 |
!isa(V)) { |
7382 |
!isa(V)) { |
| 7383 |
SDValue ReplOpOps[] = { ISR.getOperand(0), V, ISR.getOperand(2) }; |
7383 |
SDValue ReplOpOps[] = { ISR.getOperand(0), V, ISR.getOperand(2) }; |
| 7384 |
SDNode *ReplOp = |
7384 |
SDNode *ReplOp = |
| 7385 |
CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, SDLoc(V), |
7385 |
CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, SDLoc(V), |
| 7386 |
ISR.getNode()->getVTList(), ReplOpOps); |
7386 |
ISR.getNode()->getVTList(), ReplOpOps); |
| 7387 |
Ops.push_back(SDValue(ReplOp, 0)); |
7387 |
Ops.push_back(SDValue(ReplOp, 0)); |
| 7388 |
} else { |
7388 |
} else { |
| 7389 |
Ops.push_back(V); |
7389 |
Ops.push_back(V); |
| 7390 |
} |
7390 |
} |
| 7391 |
} |
7391 |
} |
| 7392 |
|
7392 |
|
| 7393 |
// Because all to-be-promoted nodes only have users that are other |
7393 |
// Because all to-be-promoted nodes only have users that are other |
| 7394 |
// promoted nodes (or the original INSERT_SUBREG), we can safely replace |
7394 |
// promoted nodes (or the original INSERT_SUBREG), we can safely replace |
| 7395 |
// the i32 result value type with i64. |
7395 |
// the i32 result value type with i64. |
| 7396 |
|
7396 |
|
| 7397 |
SmallVector NewVTs; |
7397 |
SmallVector NewVTs; |
| 7398 |
SDVTList VTs = PN->getVTList(); |
7398 |
SDVTList VTs = PN->getVTList(); |
| 7399 |
for (unsigned i = 0, ie = VTs.NumVTs; i != ie; ++i) |
7399 |
for (unsigned i = 0, ie = VTs.NumVTs; i != ie; ++i) |
| 7400 |
if (VTs.VTs[i] == MVT::i32) |
7400 |
if (VTs.VTs[i] == MVT::i32) |
| 7401 |
NewVTs.push_back(MVT::i64); |
7401 |
NewVTs.push_back(MVT::i64); |
| 7402 |
else |
7402 |
else |
| 7403 |
NewVTs.push_back(VTs.VTs[i]); |
7403 |
NewVTs.push_back(VTs.VTs[i]); |
| 7404 |
|
7404 |
|
| 7405 |
LLVM_DEBUG(dbgs() << "PPC64 ZExt Peephole morphing:\nOld: "); |
7405 |
LLVM_DEBUG(dbgs() << "PPC64 ZExt Peephole morphing:\nOld: "); |
| 7406 |
LLVM_DEBUG(PN->dump(CurDAG)); |
7406 |
LLVM_DEBUG(PN->dump(CurDAG)); |
| 7407 |
|
7407 |
|
| 7408 |
CurDAG->SelectNodeTo(PN, NewOpcode, CurDAG->getVTList(NewVTs), Ops); |
7408 |
CurDAG->SelectNodeTo(PN, NewOpcode, CurDAG->getVTList(NewVTs), Ops); |
| 7409 |
|
7409 |
|
| 7410 |
LLVM_DEBUG(dbgs() << "\nNew: "); |
7410 |
LLVM_DEBUG(dbgs() << "\nNew: "); |
| 7411 |
LLVM_DEBUG(PN->dump(CurDAG)); |
7411 |
LLVM_DEBUG(PN->dump(CurDAG)); |
| 7412 |
LLVM_DEBUG(dbgs() << "\n"); |
7412 |
LLVM_DEBUG(dbgs() << "\n"); |
| 7413 |
} |
7413 |
} |
| 7414 |
|
7414 |
|
| 7415 |
// Now we replace the original zero extend and its associated INSERT_SUBREG |
7415 |
// Now we replace the original zero extend and its associated INSERT_SUBREG |
| 7416 |
// with the value feeding the INSERT_SUBREG (which has now been promoted to |
7416 |
// with the value feeding the INSERT_SUBREG (which has now been promoted to |
| 7417 |
// return an i64). |
7417 |
// return an i64). |
| 7418 |
|
7418 |
|
| 7419 |
LLVM_DEBUG(dbgs() << "PPC64 ZExt Peephole replacing:\nOld: "); |
7419 |
LLVM_DEBUG(dbgs() << "PPC64 ZExt Peephole replacing:\nOld: "); |
| 7420 |
LLVM_DEBUG(N->dump(CurDAG)); |
7420 |
LLVM_DEBUG(N->dump(CurDAG)); |
| 7421 |
LLVM_DEBUG(dbgs() << "\nNew: "); |
7421 |
LLVM_DEBUG(dbgs() << "\nNew: "); |
| 7422 |
LLVM_DEBUG(Op32.getNode()->dump(CurDAG)); |
7422 |
LLVM_DEBUG(Op32.getNode()->dump(CurDAG)); |
| 7423 |
LLVM_DEBUG(dbgs() << "\n"); |
7423 |
LLVM_DEBUG(dbgs() << "\n"); |
| 7424 |
|
7424 |
|
| 7425 |
ReplaceUses(N, Op32.getNode()); |
7425 |
ReplaceUses(N, Op32.getNode()); |
| 7426 |
} |
7426 |
} |
| 7427 |
|
7427 |
|
| 7428 |
if (MadeChange) |
7428 |
if (MadeChange) |
| 7429 |
CurDAG->RemoveDeadNodes(); |
7429 |
CurDAG->RemoveDeadNodes(); |
| 7430 |
} |
7430 |
} |
| 7431 |
|
7431 |
|
| 7432 |
static bool isVSXSwap(SDValue N) { |
7432 |
static bool isVSXSwap(SDValue N) { |
| 7433 |
if (!N->isMachineOpcode()) |
7433 |
if (!N->isMachineOpcode()) |
| 7434 |
return false; |
7434 |
return false; |
| 7435 |
unsigned Opc = N->getMachineOpcode(); |
7435 |
unsigned Opc = N->getMachineOpcode(); |
| 7436 |
|
7436 |
|
| 7437 |
// Single-operand XXPERMDI or the regular XXPERMDI/XXSLDWI where the immediate |
7437 |
// Single-operand XXPERMDI or the regular XXPERMDI/XXSLDWI where the immediate |
| 7438 |
// operand is 2. |
7438 |
// operand is 2. |
| 7439 |
if (Opc == PPC::XXPERMDIs) { |
7439 |
if (Opc == PPC::XXPERMDIs) { |
| 7440 |
return isa(N->getOperand(1)) && |
7440 |
return isa(N->getOperand(1)) && |
| 7441 |
N->getConstantOperandVal(1) == 2; |
7441 |
N->getConstantOperandVal(1) == 2; |
| 7442 |
} else if (Opc == PPC::XXPERMDI || Opc == PPC::XXSLDWI) { |
7442 |
} else if (Opc == PPC::XXPERMDI || Opc == PPC::XXSLDWI) { |
| 7443 |
return N->getOperand(0) == N->getOperand(1) && |
7443 |
return N->getOperand(0) == N->getOperand(1) && |
| 7444 |
isa(N->getOperand(2)) && |
7444 |
isa(N->getOperand(2)) && |
| 7445 |
N->getConstantOperandVal(2) == 2; |
7445 |
N->getConstantOperandVal(2) == 2; |
| 7446 |
} |
7446 |
} |
| 7447 |
|
7447 |
|
| 7448 |
return false; |
7448 |
return false; |
| 7449 |
} |
7449 |
} |
| 7450 |
|
7450 |
|
| 7451 |
// TODO: Make this complete and replace with a table-gen bit. |
7451 |
// TODO: Make this complete and replace with a table-gen bit. |
| 7452 |
static bool isLaneInsensitive(SDValue N) { |
7452 |
static bool isLaneInsensitive(SDValue N) { |
| 7453 |
if (!N->isMachineOpcode()) |
7453 |
if (!N->isMachineOpcode()) |
| 7454 |
return false; |
7454 |
return false; |
| 7455 |
unsigned Opc = N->getMachineOpcode(); |
7455 |
unsigned Opc = N->getMachineOpcode(); |
| 7456 |
|
7456 |
|
| 7457 |
switch (Opc) { |
7457 |
switch (Opc) { |
| 7458 |
default: |
7458 |
default: |
| 7459 |
return false; |
7459 |
return false; |
| 7460 |
case PPC::VAVGSB: |
7460 |
case PPC::VAVGSB: |
| 7461 |
case PPC::VAVGUB: |
7461 |
case PPC::VAVGUB: |
| 7462 |
case PPC::VAVGSH: |
7462 |
case PPC::VAVGSH: |
| 7463 |
case PPC::VAVGUH: |
7463 |
case PPC::VAVGUH: |
| 7464 |
case PPC::VAVGSW: |
7464 |
case PPC::VAVGSW: |
| 7465 |
case PPC::VAVGUW: |
7465 |
case PPC::VAVGUW: |
| 7466 |
case PPC::VMAXFP: |
7466 |
case PPC::VMAXFP: |
| 7467 |
case PPC::VMAXSB: |
7467 |
case PPC::VMAXSB: |
| 7468 |
case PPC::VMAXUB: |
7468 |
case PPC::VMAXUB: |
| 7469 |
case PPC::VMAXSH: |
7469 |
case PPC::VMAXSH: |
| 7470 |
case PPC::VMAXUH: |
7470 |
case PPC::VMAXUH: |
| 7471 |
case PPC::VMAXSW: |
7471 |
case PPC::VMAXSW: |
| 7472 |
case PPC::VMAXUW: |
7472 |
case PPC::VMAXUW: |
| 7473 |
case PPC::VMINFP: |
7473 |
case PPC::VMINFP: |
| 7474 |
case PPC::VMINSB: |
7474 |
case PPC::VMINSB: |
| 7475 |
case PPC::VMINUB: |
7475 |
case PPC::VMINUB: |
| 7476 |
case PPC::VMINSH: |
7476 |
case PPC::VMINSH: |
| 7477 |
case PPC::VMINUH: |
7477 |
case PPC::VMINUH: |
| 7478 |
case PPC::VMINSW: |
7478 |
case PPC::VMINSW: |
| 7479 |
case PPC::VMINUW: |
7479 |
case PPC::VMINUW: |
| 7480 |
case PPC::VADDFP: |
7480 |
case PPC::VADDFP: |
| 7481 |
case PPC::VADDUBM: |
7481 |
case PPC::VADDUBM: |
| 7482 |
case PPC::VADDUHM: |
7482 |
case PPC::VADDUHM: |
| 7483 |
case PPC::VADDUWM: |
7483 |
case PPC::VADDUWM: |
| 7484 |
case PPC::VSUBFP: |
7484 |
case PPC::VSUBFP: |
| 7485 |
case PPC::VSUBUBM: |
7485 |
case PPC::VSUBUBM: |
| 7486 |
case PPC::VSUBUHM: |
7486 |
case PPC::VSUBUHM: |
| 7487 |
case PPC::VSUBUWM: |
7487 |
case PPC::VSUBUWM: |
| 7488 |
case PPC::VAND: |
7488 |
case PPC::VAND: |
| 7489 |
case PPC::VANDC: |
7489 |
case PPC::VANDC: |
| 7490 |
case PPC::VOR: |
7490 |
case PPC::VOR: |
| 7491 |
case PPC::VORC: |
7491 |
case PPC::VORC: |
| 7492 |
case PPC::VXOR: |
7492 |
case PPC::VXOR: |
| 7493 |
case PPC::VNOR: |
7493 |
case PPC::VNOR: |
| 7494 |
case PPC::VMULUWM: |
7494 |
case PPC::VMULUWM: |
| 7495 |
return true; |
7495 |
return true; |
| 7496 |
} |
7496 |
} |
| 7497 |
} |
7497 |
} |
| 7498 |
|
7498 |
|
| 7499 |
// Try to simplify (xxswap (vec-op (xxswap) (xxswap))) where vec-op is |
7499 |
// Try to simplify (xxswap (vec-op (xxswap) (xxswap))) where vec-op is |
| 7500 |
// lane-insensitive. |
7500 |
// lane-insensitive. |
| 7501 |
static void reduceVSXSwap(SDNode *N, SelectionDAG *DAG) { |
7501 |
static void reduceVSXSwap(SDNode *N, SelectionDAG *DAG) { |
| 7502 |
// Our desired xxswap might be source of COPY_TO_REGCLASS. |
7502 |
// Our desired xxswap might be source of COPY_TO_REGCLASS. |
| 7503 |
// TODO: Can we put this a common method for DAG? |
7503 |
// TODO: Can we put this a common method for DAG? |
| 7504 |
auto SkipRCCopy = [](SDValue V) { |
7504 |
auto SkipRCCopy = [](SDValue V) { |
| 7505 |
while (V->isMachineOpcode() && |
7505 |
while (V->isMachineOpcode() && |
| 7506 |
V->getMachineOpcode() == TargetOpcode::COPY_TO_REGCLASS) { |
7506 |
V->getMachineOpcode() == TargetOpcode::COPY_TO_REGCLASS) { |
| 7507 |
// All values in the chain should have single use. |
7507 |
// All values in the chain should have single use. |
| 7508 |
if (V->use_empty() || !V->use_begin()->isOnlyUserOf(V.getNode())) |
7508 |
if (V->use_empty() || !V->use_begin()->isOnlyUserOf(V.getNode())) |
| 7509 |
return SDValue(); |
7509 |
return SDValue(); |
| 7510 |
V = V->getOperand(0); |
7510 |
V = V->getOperand(0); |
| 7511 |
} |
7511 |
} |
| 7512 |
return V.hasOneUse() ? V : SDValue(); |
7512 |
return V.hasOneUse() ? V : SDValue(); |
| 7513 |
}; |
7513 |
}; |
| 7514 |
|
7514 |
|
| 7515 |
SDValue VecOp = SkipRCCopy(N->getOperand(0)); |
7515 |
SDValue VecOp = SkipRCCopy(N->getOperand(0)); |
| 7516 |
if (!VecOp || !isLaneInsensitive(VecOp)) |
7516 |
if (!VecOp || !isLaneInsensitive(VecOp)) |
| 7517 |
return; |
7517 |
return; |
| 7518 |
|
7518 |
|
| 7519 |
SDValue LHS = SkipRCCopy(VecOp.getOperand(0)), |
7519 |
SDValue LHS = SkipRCCopy(VecOp.getOperand(0)), |
| 7520 |
RHS = SkipRCCopy(VecOp.getOperand(1)); |
7520 |
RHS = SkipRCCopy(VecOp.getOperand(1)); |
| 7521 |
if (!LHS || !RHS || !isVSXSwap(LHS) || !isVSXSwap(RHS)) |
7521 |
if (!LHS || !RHS || !isVSXSwap(LHS) || !isVSXSwap(RHS)) |
| 7522 |
return; |
7522 |
return; |
| 7523 |
|
7523 |
|
| 7524 |
// These swaps may still have chain-uses here, count on dead code elimination |
7524 |
// These swaps may still have chain-uses here, count on dead code elimination |
| 7525 |
// in following passes to remove them. |
7525 |
// in following passes to remove them. |
| 7526 |
DAG->ReplaceAllUsesOfValueWith(LHS, LHS.getOperand(0)); |
7526 |
DAG->ReplaceAllUsesOfValueWith(LHS, LHS.getOperand(0)); |
| 7527 |
DAG->ReplaceAllUsesOfValueWith(RHS, RHS.getOperand(0)); |
7527 |
DAG->ReplaceAllUsesOfValueWith(RHS, RHS.getOperand(0)); |
| 7528 |
DAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), N->getOperand(0)); |
7528 |
DAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), N->getOperand(0)); |
| 7529 |
} |
7529 |
} |
| 7530 |
|
7530 |
|
| 7531 |
void PPCDAGToDAGISel::PeepholePPC64() { |
7531 |
void PPCDAGToDAGISel::PeepholePPC64() { |
| 7532 |
SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); |
7532 |
SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); |
| 7533 |
|
7533 |
|
| 7534 |
while (Position != CurDAG->allnodes_begin()) { |
7534 |
while (Position != CurDAG->allnodes_begin()) { |
| 7535 |
SDNode *N = &*--Position; |
7535 |
SDNode *N = &*--Position; |
| 7536 |
// Skip dead nodes and any non-machine opcodes. |
7536 |
// Skip dead nodes and any non-machine opcodes. |
| 7537 |
if (N->use_empty() || !N->isMachineOpcode()) |
7537 |
if (N->use_empty() || !N->isMachineOpcode()) |
| 7538 |
continue; |
7538 |
continue; |
| 7539 |
|
7539 |
|
| 7540 |
if (isVSXSwap(SDValue(N, 0))) |
7540 |
if (isVSXSwap(SDValue(N, 0))) |
| 7541 |
reduceVSXSwap(N, CurDAG); |
7541 |
reduceVSXSwap(N, CurDAG); |
| 7542 |
|
7542 |
|
| 7543 |
unsigned FirstOp; |
7543 |
unsigned FirstOp; |
| 7544 |
unsigned StorageOpcode = N->getMachineOpcode(); |
7544 |
unsigned StorageOpcode = N->getMachineOpcode(); |
| 7545 |
bool RequiresMod4Offset = false; |
7545 |
bool RequiresMod4Offset = false; |
| 7546 |
|
7546 |
|
| 7547 |
switch (StorageOpcode) { |
7547 |
switch (StorageOpcode) { |
| 7548 |
default: continue; |
7548 |
default: continue; |
| 7549 |
|
7549 |
|
| 7550 |
case PPC::LWA: |
7550 |
case PPC::LWA: |
| 7551 |
case PPC::LD: |
7551 |
case PPC::LD: |
| 7552 |
case PPC::DFLOADf64: |
7552 |
case PPC::DFLOADf64: |
| 7553 |
case PPC::DFLOADf32: |
7553 |
case PPC::DFLOADf32: |
| 7554 |
RequiresMod4Offset = true; |
7554 |
RequiresMod4Offset = true; |
| 7555 |
[[fallthrough]]; |
7555 |
[[fallthrough]]; |
| 7556 |
case PPC::LBZ: |
7556 |
case PPC::LBZ: |
| 7557 |
case PPC::LBZ8: |
7557 |
case PPC::LBZ8: |
| 7558 |
case PPC::LFD: |
7558 |
case PPC::LFD: |
| 7559 |
case PPC::LFS: |
7559 |
case PPC::LFS: |
| 7560 |
case PPC::LHA: |
7560 |
case PPC::LHA: |
| 7561 |
case PPC::LHA8: |
7561 |
case PPC::LHA8: |
| 7562 |
case PPC::LHZ: |
7562 |
case PPC::LHZ: |
| 7563 |
case PPC::LHZ8: |
7563 |
case PPC::LHZ8: |
| 7564 |
case PPC::LWZ: |
7564 |
case PPC::LWZ: |
| 7565 |
case PPC::LWZ8: |
7565 |
case PPC::LWZ8: |
| 7566 |
FirstOp = 0; |
7566 |
FirstOp = 0; |
| 7567 |
break; |
7567 |
break; |
| 7568 |
|
7568 |
|
| 7569 |
case PPC::STD: |
7569 |
case PPC::STD: |
| 7570 |
case PPC::DFSTOREf64: |
7570 |
case PPC::DFSTOREf64: |
| 7571 |
case PPC::DFSTOREf32: |
7571 |
case PPC::DFSTOREf32: |
| 7572 |
RequiresMod4Offset = true; |
7572 |
RequiresMod4Offset = true; |
| 7573 |
[[fallthrough]]; |
7573 |
[[fallthrough]]; |
| 7574 |
case PPC::STB: |
7574 |
case PPC::STB: |
| 7575 |
case PPC::STB8: |
7575 |
case PPC::STB8: |
| 7576 |
case PPC::STFD: |
7576 |
case PPC::STFD: |
| 7577 |
case PPC::STFS: |
7577 |
case PPC::STFS: |
| 7578 |
case PPC::STH: |
7578 |
case PPC::STH: |
| 7579 |
case PPC::STH8: |
7579 |
case PPC::STH8: |
| 7580 |
case PPC::STW: |
7580 |
case PPC::STW: |
| 7581 |
case PPC::STW8: |
7581 |
case PPC::STW8: |
| 7582 |
FirstOp = 1; |
7582 |
FirstOp = 1; |
| 7583 |
break; |
7583 |
break; |
| 7584 |
} |
7584 |
} |
| 7585 |
|
7585 |
|
| 7586 |
// If this is a load or store with a zero offset, or within the alignment, |
7586 |
// If this is a load or store with a zero offset, or within the alignment, |
| 7587 |
// we may be able to fold an add-immediate into the memory operation. |
7587 |
// we may be able to fold an add-immediate into the memory operation. |
| 7588 |
// The check against alignment is below, as it can't occur until we check |
7588 |
// The check against alignment is below, as it can't occur until we check |
| 7589 |
// the arguments to N |
7589 |
// the arguments to N |
| 7590 |
if (!isa(N->getOperand(FirstOp))) |
7590 |
if (!isa(N->getOperand(FirstOp))) |
| 7591 |
continue; |
7591 |
continue; |
| 7592 |
|
7592 |
|
| 7593 |
SDValue Base = N->getOperand(FirstOp + 1); |
7593 |
SDValue Base = N->getOperand(FirstOp + 1); |
| 7594 |
if (!Base.isMachineOpcode()) |
7594 |
if (!Base.isMachineOpcode()) |
| 7595 |
continue; |
7595 |
continue; |
| 7596 |
|
7596 |
|
| 7597 |
unsigned Flags = 0; |
7597 |
unsigned Flags = 0; |
| 7598 |
bool ReplaceFlags = true; |
7598 |
bool ReplaceFlags = true; |
| 7599 |
|
7599 |
|
| 7600 |
// When the feeding operation is an add-immediate of some sort, |
7600 |
// When the feeding operation is an add-immediate of some sort, |
| 7601 |
// determine whether we need to add relocation information to the |
7601 |
// determine whether we need to add relocation information to the |
| 7602 |
// target flags on the immediate operand when we fold it into the |
7602 |
// target flags on the immediate operand when we fold it into the |
| 7603 |
// load instruction. |
7603 |
// load instruction. |
| 7604 |
// |
7604 |
// |
| 7605 |
// For something like ADDItocL, the relocation information is |
7605 |
// For something like ADDItocL, the relocation information is |
| 7606 |
// inferred from the opcode; when we process it in the AsmPrinter, |
7606 |
// inferred from the opcode; when we process it in the AsmPrinter, |
| 7607 |
// we add the necessary relocation there. A load, though, can receive |
7607 |
// we add the necessary relocation there. A load, though, can receive |
| 7608 |
// relocation from various flavors of ADDIxxx, so we need to carry |
7608 |
// relocation from various flavors of ADDIxxx, so we need to carry |
| 7609 |
// the relocation information in the target flags. |
7609 |
// the relocation information in the target flags. |
| 7610 |
switch (Base.getMachineOpcode()) { |
7610 |
switch (Base.getMachineOpcode()) { |
| 7611 |
default: continue; |
7611 |
default: continue; |
| 7612 |
|
7612 |
|
| 7613 |
case PPC::ADDI8: |
7613 |
case PPC::ADDI8: |
| 7614 |
case PPC::ADDI: |
7614 |
case PPC::ADDI: |
| 7615 |
// In some cases (such as TLS) the relocation information |
7615 |
// In some cases (such as TLS) the relocation information |
| 7616 |
// is already in place on the operand, so copying the operand |
7616 |
// is already in place on the operand, so copying the operand |
| 7617 |
// is sufficient. |
7617 |
// is sufficient. |
| 7618 |
ReplaceFlags = false; |
7618 |
ReplaceFlags = false; |
| 7619 |
// For these cases, the immediate may not be divisible by 4, in |
7619 |
// For these cases, the immediate may not be divisible by 4, in |
| 7620 |
// which case the fold is illegal for DS-form instructions. (The |
7620 |
// which case the fold is illegal for DS-form instructions. (The |
| 7621 |
// other cases provide aligned addresses and are always safe.) |
7621 |
// other cases provide aligned addresses and are always safe.) |
| 7622 |
if (RequiresMod4Offset && |
7622 |
if (RequiresMod4Offset && |
| 7623 |
(!isa(Base.getOperand(1)) || |
7623 |
(!isa(Base.getOperand(1)) || |
| 7624 |
Base.getConstantOperandVal(1) % 4 != 0)) |
7624 |
Base.getConstantOperandVal(1) % 4 != 0)) |
| 7625 |
continue; |
7625 |
continue; |
| 7626 |
break; |
7626 |
break; |
| 7627 |
case PPC::ADDIdtprelL: |
7627 |
case PPC::ADDIdtprelL: |
| 7628 |
Flags = PPCII::MO_DTPREL_LO; |
7628 |
Flags = PPCII::MO_DTPREL_LO; |
| 7629 |
break; |
7629 |
break; |
| 7630 |
case PPC::ADDItlsldL: |
7630 |
case PPC::ADDItlsldL: |
| 7631 |
Flags = PPCII::MO_TLSLD_LO; |
7631 |
Flags = PPCII::MO_TLSLD_LO; |
| 7632 |
break; |
7632 |
break; |
| 7633 |
case PPC::ADDItocL: |
7633 |
case PPC::ADDItocL: |
| 7634 |
Flags = PPCII::MO_TOC_LO; |
7634 |
Flags = PPCII::MO_TOC_LO; |
| 7635 |
break; |
7635 |
break; |
| 7636 |
case PPC::ADDItoc: |
7636 |
case PPC::ADDItoc: |
| 7637 |
case PPC::ADDItoc8: |
7637 |
case PPC::ADDItoc8: |
| 7638 |
if (RequiresMod4Offset) { |
7638 |
if (RequiresMod4Offset) { |
| 7639 |
if (GlobalAddressSDNode *GA = |
7639 |
if (GlobalAddressSDNode *GA = |
| 7640 |
dyn_cast(Base.getOperand(0))) { |
7640 |
dyn_cast(Base.getOperand(0))) { |
| 7641 |
const GlobalValue *GV = GA->getGlobal(); |
7641 |
const GlobalValue *GV = GA->getGlobal(); |
| 7642 |
Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout()); |
7642 |
Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout()); |
| 7643 |
// XMC_TD global that is underaligned being accessed with a DS form |
7643 |
// XMC_TD global that is underaligned being accessed with a DS form |
| 7644 |
// instruction. |
7644 |
// instruction. |
| 7645 |
if (Alignment < 4) |
7645 |
if (Alignment < 4) |
| 7646 |
continue; |
7646 |
continue; |
| 7647 |
} |
7647 |
} |
| 7648 |
} |
7648 |
} |
| 7649 |
break; |
7649 |
break; |
| 7650 |
} |
7650 |
} |
| 7651 |
|
7651 |
|
| 7652 |
SDValue ImmOpnd = Base.getOperand(1); |
7652 |
SDValue ImmOpnd = Base.getOperand(1); |
| 7653 |
|
7653 |
|
| 7654 |
// On PPC64, the TOC base pointer is guaranteed by the ABI only to have |
7654 |
// On PPC64, the TOC base pointer is guaranteed by the ABI only to have |
| 7655 |
// 8-byte alignment, and so we can only use offsets less than 8 (otherwise, |
7655 |
// 8-byte alignment, and so we can only use offsets less than 8 (otherwise, |
| 7656 |
// we might have needed different @ha relocation values for the offset |
7656 |
// we might have needed different @ha relocation values for the offset |
| 7657 |
// pointers). |
7657 |
// pointers). |
| 7658 |
int MaxDisplacement = 7; |
7658 |
int MaxDisplacement = 7; |
| 7659 |
if (GlobalAddressSDNode *GA = dyn_cast(ImmOpnd)) { |
7659 |
if (GlobalAddressSDNode *GA = dyn_cast(ImmOpnd)) { |
| 7660 |
const GlobalValue *GV = GA->getGlobal(); |
7660 |
const GlobalValue *GV = GA->getGlobal(); |
| 7661 |
Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout()); |
7661 |
Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout()); |
| 7662 |
MaxDisplacement = std::min((int)Alignment.value() - 1, MaxDisplacement); |
7662 |
MaxDisplacement = std::min((int)Alignment.value() - 1, MaxDisplacement); |
| 7663 |
} |
7663 |
} |
| 7664 |
|
7664 |
|
| 7665 |
bool UpdateHBase = false; |
7665 |
bool UpdateHBase = false; |
| 7666 |
SDValue HBase = Base.getOperand(0); |
7666 |
SDValue HBase = Base.getOperand(0); |
| 7667 |
|
7667 |
|
| 7668 |
int Offset = N->getConstantOperandVal(FirstOp); |
7668 |
int Offset = N->getConstantOperandVal(FirstOp); |
| 7669 |
if (ReplaceFlags) { |
7669 |
if (ReplaceFlags) { |
| 7670 |
if (Offset < 0 || Offset > MaxDisplacement) { |
7670 |
if (Offset < 0 || Offset > MaxDisplacement) { |
| 7671 |
// If we have a addi(toc@l)/addis(toc@ha) pair, and the addis has only |
7671 |
// If we have a addi(toc@l)/addis(toc@ha) pair, and the addis has only |
| 7672 |
// one use, then we can do this for any offset, we just need to also |
7672 |
// one use, then we can do this for any offset, we just need to also |
| 7673 |
// update the offset (i.e. the symbol addend) on the addis also. |
7673 |
// update the offset (i.e. the symbol addend) on the addis also. |
| 7674 |
if (Base.getMachineOpcode() != PPC::ADDItocL) |
7674 |
if (Base.getMachineOpcode() != PPC::ADDItocL) |
| 7675 |
continue; |
7675 |
continue; |
| 7676 |
|
7676 |
|
| 7677 |
if (!HBase.isMachineOpcode() || |
7677 |
if (!HBase.isMachineOpcode() || |
| 7678 |
HBase.getMachineOpcode() != PPC::ADDIStocHA8) |
7678 |
HBase.getMachineOpcode() != PPC::ADDIStocHA8) |
| 7679 |
continue; |
7679 |
continue; |
| 7680 |
|
7680 |
|
| 7681 |
if (!Base.hasOneUse() || !HBase.hasOneUse()) |
7681 |
if (!Base.hasOneUse() || !HBase.hasOneUse()) |
| 7682 |
continue; |
7682 |
continue; |
| 7683 |
|
7683 |
|
| 7684 |
SDValue HImmOpnd = HBase.getOperand(1); |
7684 |
SDValue HImmOpnd = HBase.getOperand(1); |
| 7685 |
if (HImmOpnd != ImmOpnd) |
7685 |
if (HImmOpnd != ImmOpnd) |
| 7686 |
continue; |
7686 |
continue; |
| 7687 |
|
7687 |
|
| 7688 |
UpdateHBase = true; |
7688 |
UpdateHBase = true; |
| 7689 |
} |
7689 |
} |
| 7690 |
} else { |
7690 |
} else { |
| 7691 |
// If we're directly folding the addend from an addi instruction, then: |
7691 |
// If we're directly folding the addend from an addi instruction, then: |
| 7692 |
// 1. In general, the offset on the memory access must be zero. |
7692 |
// 1. In general, the offset on the memory access must be zero. |
| 7693 |
// 2. If the addend is a constant, then it can be combined with a |
7693 |
// 2. If the addend is a constant, then it can be combined with a |
| 7694 |
// non-zero offset, but only if the result meets the encoding |
7694 |
// non-zero offset, but only if the result meets the encoding |
| 7695 |
// requirements. |
7695 |
// requirements. |
| 7696 |
if (auto *C = dyn_cast(ImmOpnd)) { |
7696 |
if (auto *C = dyn_cast(ImmOpnd)) { |
| 7697 |
Offset += C->getSExtValue(); |
7697 |
Offset += C->getSExtValue(); |
| 7698 |
|
7698 |
|
| 7699 |
if (RequiresMod4Offset && (Offset % 4) != 0) |
7699 |
if (RequiresMod4Offset && (Offset % 4) != 0) |
| 7700 |
continue; |
7700 |
continue; |
| 7701 |
|
7701 |
|
| 7702 |
if (!isInt<16>(Offset)) |
7702 |
if (!isInt<16>(Offset)) |
| 7703 |
continue; |
7703 |
continue; |
| 7704 |
|
7704 |
|
| 7705 |
ImmOpnd = CurDAG->getTargetConstant(Offset, SDLoc(ImmOpnd), |
7705 |
ImmOpnd = CurDAG->getTargetConstant(Offset, SDLoc(ImmOpnd), |
| 7706 |
ImmOpnd.getValueType()); |
7706 |
ImmOpnd.getValueType()); |
| 7707 |
} else if (Offset != 0) { |
7707 |
} else if (Offset != 0) { |
| 7708 |
continue; |
7708 |
continue; |
| 7709 |
} |
7709 |
} |
| 7710 |
} |
7710 |
} |
| 7711 |
|
7711 |
|
| 7712 |
// We found an opportunity. Reverse the operands from the add |
7712 |
// We found an opportunity. Reverse the operands from the add |
| 7713 |
// immediate and substitute them into the load or store. If |
7713 |
// immediate and substitute them into the load or store. If |
| 7714 |
// needed, update the target flags for the immediate operand to |
7714 |
// needed, update the target flags for the immediate operand to |
| 7715 |
// reflect the necessary relocation information. |
7715 |
// reflect the necessary relocation information. |
| 7716 |
LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: "); |
7716 |
LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: "); |
| 7717 |
LLVM_DEBUG(Base->dump(CurDAG)); |
7717 |
LLVM_DEBUG(Base->dump(CurDAG)); |
| 7718 |
LLVM_DEBUG(dbgs() << "\nN: "); |
7718 |
LLVM_DEBUG(dbgs() << "\nN: "); |
| 7719 |
LLVM_DEBUG(N->dump(CurDAG)); |
7719 |
LLVM_DEBUG(N->dump(CurDAG)); |
| 7720 |
LLVM_DEBUG(dbgs() << "\n"); |
7720 |
LLVM_DEBUG(dbgs() << "\n"); |
| 7721 |
|
7721 |
|
| 7722 |
// If the relocation information isn't already present on the |
7722 |
// If the relocation information isn't already present on the |
| 7723 |
// immediate operand, add it now. |
7723 |
// immediate operand, add it now. |
| 7724 |
if (ReplaceFlags) { |
7724 |
if (ReplaceFlags) { |
| 7725 |
if (GlobalAddressSDNode *GA = dyn_cast(ImmOpnd)) { |
7725 |
if (GlobalAddressSDNode *GA = dyn_cast(ImmOpnd)) { |
| 7726 |
SDLoc dl(GA); |
7726 |
SDLoc dl(GA); |
| 7727 |
const GlobalValue *GV = GA->getGlobal(); |
7727 |
const GlobalValue *GV = GA->getGlobal(); |
| 7728 |
Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout()); |
7728 |
Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout()); |
| 7729 |
// We can't perform this optimization for data whose alignment |
7729 |
// We can't perform this optimization for data whose alignment |
| 7730 |
// is insufficient for the instruction encoding. |
7730 |
// is insufficient for the instruction encoding. |
| 7731 |
if (Alignment < 4 && (RequiresMod4Offset || (Offset % 4) != 0)) { |
7731 |
if (Alignment < 4 && (RequiresMod4Offset || (Offset % 4) != 0)) { |
| 7732 |
LLVM_DEBUG(dbgs() << "Rejected this candidate for alignment.\n\n"); |
7732 |
LLVM_DEBUG(dbgs() << "Rejected this candidate for alignment.\n\n"); |
| 7733 |
continue; |
7733 |
continue; |
| 7734 |
} |
7734 |
} |
| 7735 |
ImmOpnd = CurDAG->getTargetGlobalAddress(GV, dl, MVT::i64, Offset, Flags); |
7735 |
ImmOpnd = CurDAG->getTargetGlobalAddress(GV, dl, MVT::i64, Offset, Flags); |
| 7736 |
} else if (ConstantPoolSDNode *CP = |
7736 |
} else if (ConstantPoolSDNode *CP = |
| 7737 |
dyn_cast(ImmOpnd)) { |
7737 |
dyn_cast(ImmOpnd)) { |
| 7738 |
const Constant *C = CP->getConstVal(); |
7738 |
const Constant *C = CP->getConstVal(); |
| 7739 |
ImmOpnd = CurDAG->getTargetConstantPool(C, MVT::i64, CP->getAlign(), |
7739 |
ImmOpnd = CurDAG->getTargetConstantPool(C, MVT::i64, CP->getAlign(), |
| 7740 |
Offset, Flags); |
7740 |
Offset, Flags); |
| 7741 |
} |
7741 |
} |
| 7742 |
} |
7742 |
} |
| 7743 |
|
7743 |
|
| 7744 |
const unsigned BaseOpcode = Base.getMachineOpcode(); |
7744 |
const unsigned BaseOpcode = Base.getMachineOpcode(); |
| 7745 |
// ADDItoc and ADDItoc8 are pseudos used exclusively by AIX small code |
7745 |
// ADDItoc and ADDItoc8 are pseudos used exclusively by AIX small code |
| 7746 |
// model when a global is defined in the TOC. |
7746 |
// model when a global is defined in the TOC. |
| 7747 |
const bool OpcodeIsAIXTocData = |
7747 |
const bool OpcodeIsAIXTocData = |
| 7748 |
BaseOpcode == PPC::ADDItoc || BaseOpcode == PPC::ADDItoc8; |
7748 |
BaseOpcode == PPC::ADDItoc || BaseOpcode == PPC::ADDItoc8; |
| 7749 |
|
7749 |
|
| 7750 |
if (FirstOp == 1) // Store |
7750 |
if (FirstOp == 1) // Store |
| 7751 |
if (OpcodeIsAIXTocData) |
7751 |
if (OpcodeIsAIXTocData) |
| 7752 |
(void)CurDAG->UpdateNodeOperands(N, N->getOperand(0), |
7752 |
(void)CurDAG->UpdateNodeOperands(N, N->getOperand(0), |
| 7753 |
Base.getOperand(0), Base.getOperand(1), |
7753 |
Base.getOperand(0), Base.getOperand(1), |
| 7754 |
N->getOperand(3)); |
7754 |
N->getOperand(3)); |
| 7755 |
else |
7755 |
else |
| 7756 |
(void)CurDAG->UpdateNodeOperands(N, N->getOperand(0), ImmOpnd, |
7756 |
(void)CurDAG->UpdateNodeOperands(N, N->getOperand(0), ImmOpnd, |
| 7757 |
Base.getOperand(0), N->getOperand(3)); |
7757 |
Base.getOperand(0), N->getOperand(3)); |
| 7758 |
else // Load |
7758 |
else // Load |
| 7759 |
if (OpcodeIsAIXTocData) |
7759 |
if (OpcodeIsAIXTocData) |
| 7760 |
(void)CurDAG->UpdateNodeOperands(N, Base.getOperand(0), |
7760 |
(void)CurDAG->UpdateNodeOperands(N, Base.getOperand(0), |
| 7761 |
Base.getOperand(1), N->getOperand(2)); |
7761 |
Base.getOperand(1), N->getOperand(2)); |
| 7762 |
else |
7762 |
else |
| 7763 |
(void)CurDAG->UpdateNodeOperands(N, ImmOpnd, Base.getOperand(0), |
7763 |
(void)CurDAG->UpdateNodeOperands(N, ImmOpnd, Base.getOperand(0), |
| 7764 |
N->getOperand(2)); |
7764 |
N->getOperand(2)); |
| 7765 |
|
7765 |
|
| 7766 |
if (UpdateHBase) |
7766 |
if (UpdateHBase) |
| 7767 |
(void)CurDAG->UpdateNodeOperands(HBase.getNode(), HBase.getOperand(0), |
7767 |
(void)CurDAG->UpdateNodeOperands(HBase.getNode(), HBase.getOperand(0), |
| 7768 |
ImmOpnd); |
7768 |
ImmOpnd); |
| 7769 |
|
7769 |
|
| 7770 |
// The add-immediate may now be dead, in which case remove it. |
7770 |
// The add-immediate may now be dead, in which case remove it. |
| 7771 |
if (Base.getNode()->use_empty()) |
7771 |
if (Base.getNode()->use_empty()) |
| 7772 |
CurDAG->RemoveDeadNode(Base.getNode()); |
7772 |
CurDAG->RemoveDeadNode(Base.getNode()); |
| 7773 |
} |
7773 |
} |
| 7774 |
} |
7774 |
} |
| 7775 |
|
7775 |
|
| 7776 |
/// createPPCISelDag - This pass converts a legalized DAG into a |
7776 |
/// createPPCISelDag - This pass converts a legalized DAG into a |
| 7777 |
/// PowerPC-specific DAG, ready for instruction scheduling. |
7777 |
/// PowerPC-specific DAG, ready for instruction scheduling. |
| 7778 |
/// |
7778 |
/// |
| 7779 |
FunctionPass *llvm::createPPCISelDag(PPCTargetMachine &TM, |
7779 |
FunctionPass *llvm::createPPCISelDag(PPCTargetMachine &TM, |
| 7780 |
CodeGenOpt::Level OptLevel) { |
7780 |
CodeGenOpt::Level OptLevel) { |
| 7781 |
return new PPCDAGToDAGISel(TM, OptLevel); |
7781 |
return new PPCDAGToDAGISel(TM, OptLevel); |
| 7782 |
} |
7782 |
} |
| 7783 |
|
7783 |
|